Complete report #1261396
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
nf_nat_setup_info
|
nf_nat |
0xffffffffc07223d2
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x2e2
|
|
2
|
__nf_nat_alloc_null_binding
|
nf_nat |
0xffffffffc0722467
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x57
|
|
3
|
nf_nat_alloc_null_binding
|
nf_nat |
0xffffffffc07224b1
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x21
|
|
4
|
nf_nat_ipv4_fn
|
nf_nat_ipv4 |
0xffffffffc071778e
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x1be
|
|
5
|
|
iptable_nat |
0xffffffffc0729020
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x20
|
|
6
|
iptable_nat_ipv4_fn
|
iptable_nat |
0xffffffffc0729018
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x18
|
|
7
|
nf_iterate
|
vmlinux |
0xffffffff93693cf8
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x98
|
|
8
|
nf_hook_slow
|
vmlinux |
0xffffffff93693de8
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0xa8
|
|
9
|
ip_local_deliver
|
vmlinux |
0xffffffff9369e456
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0xc6
|
|
10
|
|
vmlinux |
0xffffffff9369e040
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x370
|
|
11
|
ip_rcv_finish
|
vmlinux |
0xffffffff9369dd60
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x90
|
|
12
|
ip_rcv
|
vmlinux |
0xffffffff9369e730
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x2c0
|
|
13
|
|
vmlinux |
0xffffffff9369dcd0
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x40
|
|
14
|
__netif_receive_skb_core
|
vmlinux |
0xffffffff93653c19
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x729
|
|
15
|
|
vmlinux |
0xffffffff936caf00
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0xa0
|
|
16
|
|
vmlinux |
0xffffffff931082af
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x3f
|
|
17
|
__netif_receive_skb
|
vmlinux |
0xffffffff93653f18
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x18
|
|
18
|
netif_receive_skb_internal
|
vmlinux |
0xffffffff93653fa0
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x40
|
|
19
|
napi_gro_receive
|
vmlinux |
0xffffffff93654c28
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0xd8
|
|
20
|
vmxnet3_rq_rx_complete
|
vmxnet3 |
0xffffffffc02486b4
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x924
|
|
21
|
vmxnet3_poll_rx_only
|
vmxnet3 |
0xffffffffc02490e6
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x36
|
|
22
|
net_rx_action
|
vmlinux |
0xffffffff936545bf
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x26f
|
|
23
|
__do_softirq
|
vmlinux |
0xffffffff930a5695
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0xf5
|
|
24
|
run_ksoftirqd
|
vmlinux |
0xffffffff930a5858
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x38
|
|
25
|
smpboot_thread_fn
|
vmlinux |
0xffffffff930cf0e4
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x144
|
|
26
|
|
vmlinux |
0xffffffff930cefa0
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x40
|
|
27
|
kthread
|
vmlinux |
0xffffffff930c6691
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0xd1
|
|
28
|
|
vmlinux |
0xffffffff930c65c0
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x40
|
|
29
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff93792d37
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x21
|
|
30
|
|
vmlinux |
0xffffffff930c65c0
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 26s! [ksoftirqd/2:19]
Modules linked in: falcon_lsm_serviceable(PE) falcon_nf_netcontain(E) falcon_kal(E) falcon_lsm_pinned_17804(E) xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter devlink ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw ppdev gf128mul glue_helper ablk_helper vmw_balloon cryptd pcspkr sg joydev parport_pc i2c_piix4 vmw_vmci parport sch_fq_codel ip_tables ext4 mbcache jbd2 ata_generic pata_acpi vmwgfx drm_kms_helper sd_mod syscopyarea crc_t10dif sysfillrect crct10dif_generic
sysimgblt fb_sys_fops ttm nfit ahci drm libnvdimm ata_piix libahci crct10dif_pclmul crct10dif_common crc32c_intel libata serio_raw vmxnet3 vmw_pvscsi drm_panel_orientation_quirks
CPU: 2 PID: 19 Comm: ksoftirqd/2 Kdump: loaded Tainted: P EL ------------ 3.10.0-1127.18.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff9021b9c1a0e0 ti: ffff9021b9c28000 task.ti: ffff9021b9c28000
RIP: 0010:[<ffffffffc07223d2>] [<ffffffffc07223d2>] nf_nat_setup_info+0x2e2/0x320 [nf_nat]
RSP: 0018:ffff9021b9c2b930 EFLAGS: 00010246
RAX: ffff90246dc15c98 RBX: ffff90246dc15c28 RCX: 0000000000000000
RDX: ffff9020f6db7dc8 RSI: 000000003e0085d6 RDI: ffffffffc0725478
RBP: ffff9021b9c2b9d0 R08: ffff90246dc15c28 R09: ffff9021b9c2b930
R10: ffff9021b9c2b8e0 R11: ffffffffc03ff5f8 R12: ffffffff93d15c80
R13: ffff90246dc15c28 R14: ffffffffc072201c R15: ffff9021b9c2b920
FS: 0000000000000000(0000) GS:ffff90247fc80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ff3f2f01000 CR3: 0000000034dc2000 CR4: 00000000000607e0
Call Trace:
[<ffffffffc0722467>] __nf_nat_alloc_null_binding+0x57/0x80 [nf_nat]
[<ffffffffc07224b1>] nf_nat_alloc_null_binding+0x21/0x30 [nf_nat]
[<ffffffffc071778e>] nf_nat_ipv4_fn+0x1be/0x230 [nf_nat_ipv4]
[<ffffffffc0729020>] ? iptable_nat_ipv4_fn+0x20/0x20 [iptable_nat]
[<ffffffffc0729018>] iptable_nat_ipv4_fn+0x18/0x20 [iptable_nat]
[<ffffffff93693cf8>] nf_iterate+0x98/0xe0
[<ffffffff93693de8>] nf_hook_slow+0xa8/0x110
[<ffffffff9369e456>] ip_local_deliver+0xc6/0xe0
[<ffffffff9369e040>] ? ip_rcv_finish+0x370/0x370
[<ffffffff9369dd60>] ip_rcv_finish+0x90/0x370
[<ffffffff9369e730>] ip_rcv+0x2c0/0x420
[<ffffffff9369dcd0>] ? inet_del_offload+0x40/0x40
[<ffffffff93653c19>] __netif_receive_skb_core+0x729/0xa10
[<ffffffff936caf00>] ? tcp4_gro_receive+0xa0/0x1a0
[<ffffffff931082af>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffff93653f18>] __netif_receive_skb+0x18/0x60
[<ffffffff93653fa0>] netif_receive_skb_internal+0x40/0xc0
[<ffffffff93654c28>] napi_gro_receive+0xd8/0x100
[<ffffffffc02486b4>] vmxnet3_rq_rx_complete+0x924/0xef0 [vmxnet3]
[<ffffffffc02490e6>] vmxnet3_poll_rx_only+0x36/0xa0 [vmxnet3]
[<ffffffff936545bf>] net_rx_action+0x26f/0x390
[<ffffffff930a5695>] __do_softirq+0xf5/0x280
[<ffffffff930a5858>] run_ksoftirqd+0x38/0x50
[<ffffffff930cf0e4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff930cefa0>] ? lg_double_unlock+0x40/0x40
[<ffffffff930c6691>] kthread+0xd1/0xe0
[<ffffffff930c65c0>] ? insert_kthread_work+0x40/0x40
[<ffffffff93792d37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff930c65c0>] ? insert_kthread_work+0x40/0x40
Code: d2 74 56 0f b7 42 12 66 85 c0 74 4d 48 01 d0 48 89 58 10 49 8b 96 28 0b 00 00 4a 8d 14 e2 48 8b 0a 48 89 50 08 48 89 08 48 85 c9 <48> 89 02 74 04 48 89 41 08 48 c7 c7 78 54 72 c0 e8 69 61 06 d3
Complete report #1102052
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
nf_nat_setup_info
|
nf_nat |
0xffffffffc07473c5
Build id: 3.10.0-1160.el7.x86_64
|
0x2d5
|
|
2
|
__nf_nat_alloc_null_binding
|
nf_nat |
0xffffffffc0747467
Build id: 3.10.0-1160.el7.x86_64
|
0x57
|
|
3
|
nf_nat_alloc_null_binding
|
nf_nat |
0xffffffffc07474b1
Build id: 3.10.0-1160.el7.x86_64
|
0x21
|
|
4
|
nf_nat_ipv4_fn
|
nf_nat_ipv4 |
0xffffffffc073c78e
Build id: 3.10.0-1160.el7.x86_64
|
0x1be
|
|
5
|
|
iptable_nat |
0xffffffffc074e020
Build id: 3.10.0-1160.el7.x86_64
|
0x20
|
|
6
|
iptable_nat_ipv4_fn
|
iptable_nat |
0xffffffffc074e018
Build id: 3.10.0-1160.el7.x86_64
|
0x18
|
|
7
|
nf_iterate
|
vmlinux |
0xffffffffbac94e38
Build id: 3.10.0-1160.el7.x86_64
|
0x98
|
|
8
|
nf_hook_slow
|
vmlinux |
0xffffffffbac94f28
Build id: 3.10.0-1160.el7.x86_64
|
0xa8
|
|
9
|
ip_local_deliver
|
vmlinux |
0xffffffffbac9f5e6
Build id: 3.10.0-1160.el7.x86_64
|
0xc6
|
|
10
|
|
vmlinux |
0xffffffffbac9f1d0
Build id: 3.10.0-1160.el7.x86_64
|
0x370
|
|
11
|
ip_rcv_finish
|
vmlinux |
0xffffffffbac9eef0
Build id: 3.10.0-1160.el7.x86_64
|
0x90
|
|
12
|
ip_rcv
|
vmlinux |
0xffffffffbac9f8c0
Build id: 3.10.0-1160.el7.x86_64
|
0x2c0
|
|
13
|
|
vmlinux |
0xffffffffbac9ee60
Build id: 3.10.0-1160.el7.x86_64
|
0x40
|
|
14
|
__netif_receive_skb_core
|
vmlinux |
0xffffffffbac54c19
Build id: 3.10.0-1160.el7.x86_64
|
0x729
|
|
15
|
|
vmlinux |
0xffffffffbaccc100
Build id: 3.10.0-1160.el7.x86_64
|
0x80
|
|
16
|
|
vmlinux |
0xffffffffbacddd99
Build id: 3.10.0-1160.el7.x86_64
|
0x249
|
|
17
|
__netif_receive_skb
|
vmlinux |
0xffffffffbac54f18
Build id: 3.10.0-1160.el7.x86_64
|
0x18
|
|
18
|
netif_receive_skb_internal
|
vmlinux |
0xffffffffbac54fa0
Build id: 3.10.0-1160.el7.x86_64
|
0x40
|
|
19
|
napi_gro_receive
|
vmlinux |
0xffffffffbac55c28
Build id: 3.10.0-1160.el7.x86_64
|
0xd8
|
|
20
|
e1000_clean_rx_irq
|
e1000 |
0xffffffffc024545b
Build id: 3.10.0-1160.el7.x86_64
|
0x2bb
|
|
21
|
e1000_clean
|
e1000 |
0xffffffffc0246100
Build id: 3.10.0-1160.el7.x86_64
|
0x280
|
|
22
|
|
vmlinux |
0xffffffffba6e43cc
Build id: 3.10.0-1160.el7.x86_64
|
0x14c
|
|
23
|
net_rx_action
|
vmlinux |
0xffffffffbac555bf
Build id: 3.10.0-1160.el7.x86_64
|
0x26f
|
|
24
|
__do_softirq
|
vmlinux |
0xffffffffba6a4b95
Build id: 3.10.0-1160.el7.x86_64
|
0xf5
|
|
25
|
run_ksoftirqd
|
vmlinux |
0xffffffffba6a4d58
Build id: 3.10.0-1160.el7.x86_64
|
0x38
|
|
26
|
smpboot_thread_fn
|
vmlinux |
0xffffffffba6ce674
Build id: 3.10.0-1160.el7.x86_64
|
0x144
|
|
27
|
|
vmlinux |
0xffffffffba6ce530
Build id: 3.10.0-1160.el7.x86_64
|
0x40
|
|
28
|
kthread
|
vmlinux |
0xffffffffba6c5c21
Build id: 3.10.0-1160.el7.x86_64
|
0xd1
|
|
29
|
|
vmlinux |
0xffffffffba6c5b50
Build id: 3.10.0-1160.el7.x86_64
|
0x40
|
|
30
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffbad93df7
Build id: 3.10.0-1160.el7.x86_64
|
0x21
|
|
31
|
|
vmlinux |
0xffffffffba6c5b50
Build id: 3.10.0-1160.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [ksoftirqd/0:6]
Modules linked in: tcp_lp xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr joydev ppdev vmw_balloon sg parport_pc parport vmw_vmci i2c_piix4 ip_tables ext4 mbcache jbd2 sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm nfit ata_piix mptspi drm scsi_transport_spi libnvdimm mptscsih libata mptbase
crct10dif_pclmul crct10dif_common crc32c_intel e1000 serio_raw drm_panel_orientation_quirks fuse
CPU: 0 PID: 6 Comm: ksoftirqd/0 Tainted: G L ------------ 3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff8c56bd765280 ti: ffff8c56be364000 task.ti: ffff8c56be364000
RIP: 0010:[<ffffffffc07473c5>] [<ffffffffc07473c5>] nf_nat_setup_info+0x2d5/0x320 [nf_nat]
RSP: 0018:ffff8c56be367890 EFLAGS: 00010286
RAX: ffff8c56b57fbd18 RBX: ffff8c56b57fbca8 RCX: 00000000b969b2c6
RDX: ffff8c55f6b7a730 RSI: 000000007fd1b35c RDI: ffffffffc074a478
RBP: ffff8c56be367930 R08: ffff8c56b57fbca8 R09: ffff8c56be367890
R10: ffff8c56be367840 R11: ffffffffc04305f8 R12: ffffffffbb315e40
R13: ffff8c56b57fbca8 R14: ffffffffc074701c R15: ffff8c56be367880
FS: 0000000000000000(0000) GS:ffff8c5775600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fe28bc2a480 CR3: 0000000233496000 CR4: 0000000000360ff0
Call Trace:
[<ffffffffc0747467>] __nf_nat_alloc_null_binding+0x57/0x80 [nf_nat]
[<ffffffffc07474b1>] nf_nat_alloc_null_binding+0x21/0x30 [nf_nat]
[<ffffffffc073c78e>] nf_nat_ipv4_fn+0x1be/0x230 [nf_nat_ipv4]
[<ffffffffc074e020>] ? iptable_nat_ipv4_fn+0x20/0x20 [iptable_nat]
[<ffffffffc074e018>] iptable_nat_ipv4_fn+0x18/0x20 [iptable_nat]
[<ffffffffbac94e38>] nf_iterate+0x98/0xe0
[<ffffffffbac94f28>] nf_hook_slow+0xa8/0x110
[<ffffffffbac9f5e6>] ip_local_deliver+0xc6/0xe0
[<ffffffffbac9f1d0>] ? ip_rcv_finish+0x370/0x370
[<ffffffffbac9eef0>] ip_rcv_finish+0x90/0x370
[<ffffffffbac9f8c0>] ip_rcv+0x2c0/0x420
[<ffffffffbac9ee60>] ? inet_del_offload+0x40/0x40
[<ffffffffbac54c19>] __netif_receive_skb_core+0x729/0xa10
[<ffffffffbaccc100>] ? tcp4_gro_receive+0x80/0x1a0
[<ffffffffbacddd99>] ? inet_gro_receive+0x249/0x290
[<ffffffffbac54f18>] __netif_receive_skb+0x18/0x60
[<ffffffffbac54fa0>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffbac55c28>] napi_gro_receive+0xd8/0x100
[<ffffffffc024545b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc0246100>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffffba6e43cc>] ? update_curr+0x14c/0x1e0
[<ffffffffbac555bf>] net_rx_action+0x26f/0x390
[<ffffffffba6a4b95>] __do_softirq+0xf5/0x280
[<ffffffffba6a4d58>] run_ksoftirqd+0x38/0x50
[<ffffffffba6ce674>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffba6ce530>] ? lg_double_unlock+0x40/0x40
[<ffffffffba6c5c21>] kthread+0xd1/0xe0
[<ffffffffba6c5b50>] ? insert_kthread_work+0x40/0x40
[<ffffffffbad93df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffba6c5b50>] ? insert_kthread_work+0x40/0x40
Code: 48 8b 93 e8 00 00 00 49 c1 ec 20 48 85 d2 74 56 0f b7 42 12 66 85 c0 74 4d 48 01 d0 48 89 58 10 49 8b 96 28 0b 00 00 4a 8d 14 e2 <48> 8b 0a 48 89 50 08 48 89 08 48 85 c9 48 89 02 74 04 48 89 41
Complete report #704392
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffff8d711ffd
Build id: 3.10.0-957.el7.x86_64
|
0x1d
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffff8dd5bf4b
Build id: 3.10.0-957.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock_bh
|
vmlinux |
0xffffffff8dd6a443
Build id: 3.10.0-957.el7.x86_64
|
0x33
|
|
4
|
nf_nat_setup_info
|
nf_nat |
0xffffffffc064d36a
Build id: 3.10.0-957.el7.x86_64
|
0x2aa
|
|
5
|
__nf_nat_alloc_null_binding
|
nf_nat |
0xffffffffc064d437
Build id: 3.10.0-957.el7.x86_64
|
0x57
|
|
6
|
nf_nat_alloc_null_binding
|
nf_nat |
0xffffffffc064d481
Build id: 3.10.0-957.el7.x86_64
|
0x21
|
|
7
|
nf_nat_ipv4_fn
|
nf_nat_ipv4 |
0xffffffffc065978e
Build id: 3.10.0-957.el7.x86_64
|
0x1be
|
|
8
|
|
iptable_nat |
0xffffffffc0654020
Build id: 3.10.0-957.el7.x86_64
|
0x20
|
|
9
|
iptable_nat_ipv4_fn
|
iptable_nat |
0xffffffffc0654018
Build id: 3.10.0-957.el7.x86_64
|
0x18
|
|
10
|
nf_iterate
|
vmlinux |
0xffffffff8dc784a8
Build id: 3.10.0-957.el7.x86_64
|
0x98
|
|
11
|
nf_hook_slow
|
vmlinux |
0xffffffff8dc78598
Build id: 3.10.0-957.el7.x86_64
|
0xa8
|
|
12
|
ip_local_deliver
|
vmlinux |
0xffffffff8dc82ba2
Build id: 3.10.0-957.el7.x86_64
|
0xb2
|
|
13
|
|
vmlinux |
0xffffffff8dc827a0
Build id: 3.10.0-957.el7.x86_64
|
0x370
|
|
14
|
ip_rcv_finish
|
vmlinux |
0xffffffff8dc824c0
Build id: 3.10.0-957.el7.x86_64
|
0x90
|
|
15
|
ip_rcv
|
vmlinux |
0xffffffff8dc82e79
Build id: 3.10.0-957.el7.x86_64
|
0x2b9
|
|
16
|
|
vmlinux |
0xffffffff8dc82430
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
|
17
|
__netif_receive_skb_core
|
vmlinux |
0xffffffff8dc394f9
Build id: 3.10.0-957.el7.x86_64
|
0x729
|
|
18
|
__netif_receive_skb
|
vmlinux |
0xffffffff8dc397f8
Build id: 3.10.0-957.el7.x86_64
|
0x18
|
|
19
|
process_backlog
|
vmlinux |
0xffffffff8dc3a7be
Build id: 3.10.0-957.el7.x86_64
|
0xae
|
|
20
|
net_rx_action
|
vmlinux |
0xffffffff8dc39e9f
Build id: 3.10.0-957.el7.x86_64
|
0x26f
|
|
21
|
__do_softirq
|
vmlinux |
0xffffffff8d6a0f05
Build id: 3.10.0-957.el7.x86_64
|
0xf5
|
|
22
|
run_ksoftirqd
|
vmlinux |
0xffffffff8d6a10c8
Build id: 3.10.0-957.el7.x86_64
|
0x38
|
|
23
|
smpboot_thread_fn
|
vmlinux |
0xffffffff8d6ca5c4
Build id: 3.10.0-957.el7.x86_64
|
0x144
|
|
24
|
|
vmlinux |
0xffffffff8d6ca480
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
|
25
|
kthread
|
vmlinux |
0xffffffff8d6c1c31
Build id: 3.10.0-957.el7.x86_64
|
0xd1
|
|
26
|
|
vmlinux |
0xffffffff8d6c1b60
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
|
27
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff8dd74c37
Build id: 3.10.0-957.el7.x86_64
|
0x21
|
|
28
|
|
vmlinux |
0xffffffff8d6c1b60
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#11 stuck for 22s! [xfsaild/dm-2:9475]
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 26s! [ksoftirqd/0:3]
Modules linked in: binfmt_misc veth xt_nat xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock fuse vmhgfs(OE) overlay(T) sunrpc snd_seq_midi snd_seq_midi_event sb_edac coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel snd_ens1371 lrw snd_rawmidi gf128mul ppdev glue_helper snd_ac97_codec ablk_helper cryptd ac97_bus vmw_balloon snd_seq snd_seq_device snd_pcm snd_timer snd joydev soundcore pcspkr sg vmw_vmci i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sd_mod sr_mod cdrom crc_t10dif crct10dif_generic
ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm libata mptspi scsi_transport_spi mptscsih e1000 mptbase nfit libnvdimm crct10dif_pclmul crct10dif_common crc32c_intel serio_raw drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 3 Comm: ksoftirqd/0 Kdump: loaded Tainted: G OEL ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
task: ffff8ce6fa35a080 ti: ffff8ce6f9c58000 task.ti: ffff8ce6f9c58000
RIP: 0010:[<ffffffff8d711ffd>] [<ffffffff8d711ffd>] native_queued_spin_lock_slowpath+0x1d/0x200
RSP: 0018:ffff8ce6f9c5b9e0 EFLAGS: 00000293
RAX: 0000000000000001 RBX: ffff8ce596ed17c0 RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffffffffc0650478
RBP: ffff8ce6f9c5b9e0 R08: ffff8ce8997a0928 R09: ffff8ce6f9c5ba10
R10: ffff8ce6f9c5b9c0 R11: ffffffffc016f5f8 R12: 0000000000000000
R13: ffff8ce6f9c5bac0 R14: ffff8ce6f9c5ba38 R15: 00010000f9c5b9a8
FS: 0000000000000000(0000) GS:ffff8ce9ad600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f388e269120 CR3: 00000002a7604000 CR4: 00000000003607f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff8dd5bf4b>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffff8dd6a443>] _raw_spin_lock_bh+0x33/0x40
[<ffffffffc064d36a>] nf_nat_setup_info+0x2aa/0x320 [nf_nat]
[<ffffffffc064d437>] __nf_nat_alloc_null_binding+0x57/0x80 [nf_nat]
[<ffffffffc064d481>] nf_nat_alloc_null_binding+0x21/0x30 [nf_nat]
[<ffffffffc065978e>] nf_nat_ipv4_fn+0x1be/0x230 [nf_nat_ipv4]
[<ffffffffc0654020>] ? iptable_nat_ipv4_fn+0x20/0x20 [iptable_nat]
[<ffffffffc0654018>] iptable_nat_ipv4_fn+0x18/0x20 [iptable_nat]
[<ffffffff8dc784a8>] nf_iterate+0x98/0xe0
[<ffffffff8dc78598>] nf_hook_slow+0xa8/0x110
[<ffffffff8dc82ba2>] ip_local_deliver+0xb2/0xd0
[<ffffffff8dc827a0>] ? ip_rcv_finish+0x370/0x370
[<ffffffff8dc824c0>] ip_rcv_finish+0x90/0x370
[<ffffffff8dc82e79>] ip_rcv+0x2b9/0x410
[<ffffffff8dc82430>] ? inet_del_offload+0x40/0x40
[<ffffffff8dc394f9>] __netif_receive_skb_core+0x729/0xa10
[<ffffffff8dc397f8>] __netif_receive_skb+0x18/0x60
[<ffffffff8dc3a7be>] process_backlog+0xae/0x180
[<ffffffff8dc39e9f>] net_rx_action+0x26f/0x390
[<ffffffff8d6a0f05>] __do_softirq+0xf5/0x280
[<ffffffff8d6a10c8>] run_ksoftirqd+0x38/0x50
[<ffffffff8d6ca5c4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff8d6ca480>] ? lg_double_unlock+0x40/0x40
[<ffffffff8d6c1c31>] kthread+0xd1/0xe0
[<ffffffff8d6c1b60>] ? insert_kthread_work+0x40/0x40
[<ffffffff8dd74c37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff8d6c1b60>] ? insert_kthread_work+0x40/0x40
Code: 47 fe ff ff 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 66 90 b9 01 00 00 00 8b 17 85 d2 74 0d 83 fa 03 74 08 f3 90 <8b> 17 85 d2 75 f3 89 d0 f0 0f b1 0f 39 c2 75 e3 5d 66 90 c3 0f
Complete report #491612
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffffa21122cd
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x1d
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffffa275d1cb
Build id: 3.10.0-957.12.2.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock_bh
|
vmlinux |
0xffffffffa276b6e3
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x33
|
|
4
|
nf_nat_setup_info
|
nf_nat |
0xffffffffc059a36a
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x2aa
|
|
5
|
__nf_nat_alloc_null_binding
|
nf_nat |
0xffffffffc059a437
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x57
|
|
6
|
nf_nat_alloc_null_binding
|
nf_nat |
0xffffffffc059a481
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x21
|
|
7
|
nf_nat_ipv4_fn
|
nf_nat_ipv4 |
0xffffffffc05ed78e
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x1be
|
|
8
|
|
iptable_nat |
0xffffffffc05e8020
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x20
|
|
9
|
iptable_nat_ipv4_fn
|
iptable_nat |
0xffffffffc05e8018
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x18
|
|
10
|
nf_iterate
|
vmlinux |
0xffffffffa2679528
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x98
|
|
11
|
nf_hook_slow
|
vmlinux |
0xffffffffa2679618
Build id: 3.10.0-957.12.2.el7.x86_64
|
0xa8
|
|
12
|
ip_local_deliver
|
vmlinux |
0xffffffffa2683c22
Build id: 3.10.0-957.12.2.el7.x86_64
|
0xb2
|
|
13
|
|
vmlinux |
0xffffffffa2683820
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x370
|
|
14
|
ip_rcv_finish
|
vmlinux |
0xffffffffa2683540
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x90
|
|
15
|
ip_rcv
|
vmlinux |
0xffffffffa2683ef9
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x2b9
|
|
16
|
|
vmlinux |
0xffffffffa26834b0
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x40
|
|
17
|
__netif_receive_skb_core
|
vmlinux |
0xffffffffa263a579
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x729
|
|
18
|
|
vmlinux |
0xffffffffa26aff00
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x100
|
|
19
|
|
vmlinux |
0xffffffffa26c1889
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x249
|
|
20
|
__netif_receive_skb
|
vmlinux |
0xffffffffa263a878
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x18
|
|
21
|
netif_receive_skb_internal
|
vmlinux |
0xffffffffa263a900
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x40
|
|
22
|
napi_gro_receive
|
vmlinux |
0xffffffffa263b588
Build id: 3.10.0-957.12.2.el7.x86_64
|
0xd8
|
|
23
|
vmxnet3_rq_rx_complete
|
vmxnet3 |
0xffffffffc00d89cc
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x91c
|
|
24
|
vmxnet3_poll_rx_only
|
vmxnet3 |
0xffffffffc00d90f6
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x36
|
|
25
|
net_rx_action
|
vmlinux |
0xffffffffa263af1f
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x26f
|
|
26
|
__do_softirq
|
vmlinux |
0xffffffffa20a0ff5
Build id: 3.10.0-957.12.2.el7.x86_64
|
0xf5
|
|
27
|
run_ksoftirqd
|
vmlinux |
0xffffffffa20a11b8
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x38
|
|
28
|
smpboot_thread_fn
|
vmlinux |
0xffffffffa20ca6b4
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x144
|
|
29
|
|
vmlinux |
0xffffffffa20ca570
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x40
|
|
30
|
kthread
|
vmlinux |
0xffffffffa20c1d21
Build id: 3.10.0-957.12.2.el7.x86_64
|
0xd1
|
|
31
|
|
vmlinux |
0xffffffffa20c1c50
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x40
|
|
32
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffa2775c37
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x21
|
|
33
|
|
vmlinux |
0xffffffffa20c1c50
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 23s! [ksoftirqd/2:19]
Modules linked in: nfsv3 rpcsec_gss_krb5 nfsv4 nfs fscache cmac arc4 md4 nls_utf8 cifs ccm dns_resolver fuse xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables vmw_vsock_vmci_transport vsock iptable_filter dm_mirror dm_region_hash dm_log dm_mod sb_edac coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev vmw_balloon joydev pcspkr sg parport_pc parport vmw_vmci i2c_piix4 nfsd auth_rpcgss nfs_acl lockd grace sunrpc sch_fq_codel ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi crct10dif_pclmul
crct10dif_common crc32c_intel vmwgfx drm_kms_helper serio_raw syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm vmxnet3 mptspi libata scsi_transport_spi mptscsih mptbase drm_panel_orientation_quirks floppy
CPU: 2 PID: 19 Comm: ksoftirqd/2 Tainted: G W L ------------ 3.10.0-957.12.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff8abadb264100 ti: ffff8abadb29c000 task.ti: ffff8abadb29c000
RIP: 0010:[<ffffffffa21122cd>] [<ffffffffa21122cd>] native_queued_spin_lock_slowpath+0x1d/0x200
RSP: 0018:ffff8abadb29f910 EFLAGS: 00000293
RAX: 0000000000000001 RBX: ffff8abc4baa6800 RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffffffffc059d478
RBP: ffff8abadb29f910 R08: ffff8abef4fbae68 R09: ffff8abadb29f940
R10: ffff8abadb29f8f0 R11: ffffffffc02755f8 R12: 0000000000000000
R13: ffff8abadb29f9f0 R14: ffff8abadb29f968 R15: 00010000db29f8d8
FS: 0000000000000000(0000) GS:ffff8ad15e680000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000056415be7d000 CR3: 000000093c354000 CR4: 00000000001607e0
Call Trace:
[<ffffffffa275d1cb>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffffa276b6e3>] _raw_spin_lock_bh+0x33/0x40
[<ffffffffc059a36a>] nf_nat_setup_info+0x2aa/0x320 [nf_nat]
[<ffffffffc059a437>] __nf_nat_alloc_null_binding+0x57/0x80 [nf_nat]
[<ffffffffc059a481>] nf_nat_alloc_null_binding+0x21/0x30 [nf_nat]
[<ffffffffc05ed78e>] nf_nat_ipv4_fn+0x1be/0x230 [nf_nat_ipv4]
[<ffffffffc05e8020>] ? iptable_nat_ipv4_fn+0x20/0x20 [iptable_nat]
[<ffffffffc05e8018>] iptable_nat_ipv4_fn+0x18/0x20 [iptable_nat]
[<ffffffffa2679528>] nf_iterate+0x98/0xe0
[<ffffffffa2679618>] nf_hook_slow+0xa8/0x110
[<ffffffffa2683c22>] ip_local_deliver+0xb2/0xd0
[<ffffffffa2683820>] ? ip_rcv_finish+0x370/0x370
[<ffffffffa2683540>] ip_rcv_finish+0x90/0x370
[<ffffffffa2683ef9>] ip_rcv+0x2b9/0x410
[<ffffffffa26834b0>] ? inet_del_offload+0x40/0x40
[<ffffffffa263a579>] __netif_receive_skb_core+0x729/0xa10
[<ffffffffa26aff00>] ? tcp4_gro_receive+0x100/0x1a0
[<ffffffffa26c1889>] ? inet_gro_receive+0x249/0x290
[<ffffffffa263a878>] __netif_receive_skb+0x18/0x60
[<ffffffffa263a900>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffa263b588>] napi_gro_receive+0xd8/0x100
[<ffffffffc00d89cc>] vmxnet3_rq_rx_complete+0x91c/0xef0 [vmxnet3]
[<ffffffffc00d90f6>] vmxnet3_poll_rx_only+0x36/0xa0 [vmxnet3]
[<ffffffffa263af1f>] net_rx_action+0x26f/0x390
[<ffffffffa20a0ff5>] __do_softirq+0xf5/0x280
[<ffffffffa20a11b8>] run_ksoftirqd+0x38/0x50
[<ffffffffa20ca6b4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffa20ca570>] ? lg_double_unlock+0x40/0x40
[<ffffffffa20c1d21>] kthread+0xd1/0xe0
[<ffffffffa20c1c50>] ? insert_kthread_work+0x40/0x40
[<ffffffffa2775c37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffa20c1c50>] ? insert_kthread_work+0x40/0x40
Code: 47 fe ff ff 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 66 90 b9 01 00 00 00 8b 17 85 d2 74 0d 83 fa 03 74 08 f3 90 <8b> 17 85 d2 75 f3 89 d0 f0 0f b1 0f 39 c2 75 e3 5d 66 90 c3 0f
Complete report #56811
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
nf_nat_setup_info
|
nf_nat |
0xffffffffc089c3d2
Build id: 3.10.0-1062.4.3.el7.x86_64
|
0x2e2
|
|
2
|
__nf_nat_alloc_null_binding
|
nf_nat |
0xffffffffc089c467
Build id: 3.10.0-1062.4.3.el7.x86_64
|
0x57
|
|
3
|
nf_nat_alloc_null_binding
|
nf_nat |
0xffffffffc089c4b1
Build id: 3.10.0-1062.4.3.el7.x86_64
|
0x21
|
|
4
|
nf_nat_ipv4_fn
|
nf_nat_ipv4 |
0xffffffffc077278e
Build id: 3.10.0-1062.4.3.el7.x86_64
|
0x1be
|
|
5
|
|
iptable_nat |
0xffffffffc08a8020
Build id: 3.10.0-1062.4.3.el7.x86_64
|
0x20
|
|
6
|
iptable_nat_ipv4_fn
|
iptable_nat |
0xffffffffc08a8018
Build id: 3.10.0-1062.4.3.el7.x86_64
|
0x18
|
|
7
|
nf_iterate
|
vmlinux |
0xffffffffaa08f418
Build id: 3.10.0-1062.4.3.el7.x86_64
|
0x98
|
|
8
|
nf_hook_slow
|
vmlinux |
0xffffffffaa08f508
Build id: 3.10.0-1062.4.3.el7.x86_64
|
0xa8
|
|
9
|
ip_local_deliver
|
vmlinux |
0xffffffffaa099bc6
Build id: 3.10.0-1062.4.3.el7.x86_64
|
0xc6
|
|
10
|
|
vmlinux |
0xffffffffaa0997b0
Build id: 3.10.0-1062.4.3.el7.x86_64
|
0x370
|
|
11
|
ip_rcv_finish
|
vmlinux |
0xffffffffaa0994d0
Build id: 3.10.0-1062.4.3.el7.x86_64
|
0x90
|
|
12
|
ip_rcv
|
vmlinux |
0xffffffffaa099ea0
Build id: 3.10.0-1062.4.3.el7.x86_64
|
0x2c0
|
|
13
|
|
vmlinux |
0xffffffffaa099440
Build id: 3.10.0-1062.4.3.el7.x86_64
|
0x40
|
|
14
|
__netif_receive_skb_core
|
vmlinux |
0xffffffffaa04ebd9
Build id: 3.10.0-1062.4.3.el7.x86_64
|
0x729
|
|
15
|
|
vmlinux |
0xffffffffa9b06e6f
Build id: 3.10.0-1062.4.3.el7.x86_64
|
0x3f
|
|
16
|
__netif_receive_skb
|
vmlinux |
0xffffffffaa04eed8
Build id: 3.10.0-1062.4.3.el7.x86_64
|
0x18
|
|
17
|
netif_receive_skb_internal
|
vmlinux |
0xffffffffaa04ef60
Build id: 3.10.0-1062.4.3.el7.x86_64
|
0x40
|
|
18
|
napi_gro_receive
|
vmlinux |
0xffffffffaa04fbe8
Build id: 3.10.0-1062.4.3.el7.x86_64
|
0xd8
|
|
19
|
vmxnet3_rq_rx_complete
|
vmxnet3 |
0xffffffffc03826b4
Build id: 3.10.0-1062.4.3.el7.x86_64
|
0x924
|
|
20
|
vmxnet3_poll_rx_only
|
vmxnet3 |
0xffffffffc03830e6
Build id: 3.10.0-1062.4.3.el7.x86_64
|
0x36
|
|
21
|
net_rx_action
|
vmlinux |
0xffffffffaa04f57f
Build id: 3.10.0-1062.4.3.el7.x86_64
|
0x26f
|
|
22
|
__do_softirq
|
vmlinux |
0xffffffffa9aa5305
Build id: 3.10.0-1062.4.3.el7.x86_64
|
0xf5
|
|
23
|
run_ksoftirqd
|
vmlinux |
0xffffffffa9aa54c8
Build id: 3.10.0-1062.4.3.el7.x86_64
|
0x38
|
|
24
|
smpboot_thread_fn
|
vmlinux |
0xffffffffa9acec24
Build id: 3.10.0-1062.4.3.el7.x86_64
|
0x144
|
|
25
|
|
vmlinux |
0xffffffffa9aceae0
Build id: 3.10.0-1062.4.3.el7.x86_64
|
0x40
|
|
26
|
kthread
|
vmlinux |
0xffffffffa9ac61f1
Build id: 3.10.0-1062.4.3.el7.x86_64
|
0xd1
|
|
27
|
|
vmlinux |
0xffffffffa9ac6120
Build id: 3.10.0-1062.4.3.el7.x86_64
|
0x40
|
|
28
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffaa18cd37
Build id: 3.10.0-1062.4.3.el7.x86_64
|
0x21
|
|
29
|
|
vmlinux |
0xffffffffa9ac6120
Build id: 3.10.0-1062.4.3.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [ksoftirqd/2:19]
Modules linked in: xt_nat veth xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink xt_addrtype iptable_filter iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack br_netfilter bridge stp llc cmac arc4 md4 nls_utf8 cifs ccm dns_resolver overlay(T) vmw_vsock_vmci_transport vsock sb_edac coretemp iosf_mbi crc32_pclmul ppdev ghash_clmulni_intel vmw_balloon aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr sg joydev parport_pc parport i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic vmwgfx drm_kms_helper pata_acpi syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common drm crc32c_intel serio_raw mptspi ata_piix scsi_transport_spi mptscsih libata
mptbase vmxnet3 drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 2 PID: 19 Comm: ksoftirqd/2 Kdump: loaded Tainted: G L ------------ T 3.10.0-1062.4.3.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/30/2014
task: ffff927cfb3f8000 ti: ffff927cfb3f4000 task.ti: ffff927cfb3f4000
RIP: 0010:[<ffffffffc089c3d2>] [<ffffffffc089c3d2>] nf_nat_setup_info+0x2e2/0x320 [nf_nat]
RSP: 0018:ffff927cfb3f7930 EFLAGS: 00010246
RAX: ffff927c13c56758 RBX: ffff927c13c566e8 RCX: 0000000000000000
RDX: ffff927eb1a16998 RSI: 0000000082bd32e5 RDI: ffffffffc089f478
RBP: ffff927cfb3f79d0 R08: ffff927c13c566e8 R09: ffff927cfb3f7930
R10: ffff927cfb3f78e0 R11: ffffffffc051b5f8 R12: ffffffffaa7129c0
R13: ffff927c13c566e8 R14: ffffffffc089c01c R15: ffff927cfb3f7920
FS: 0000000000000000(0000) GS:ffff927ebfd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f49a507c000 CR3: 00000003213e2000 CR4: 00000000001607e0
Call Trace:
[<ffffffffc089c467>] __nf_nat_alloc_null_binding+0x57/0x80 [nf_nat]
[<ffffffffc089c4b1>] nf_nat_alloc_null_binding+0x21/0x30 [nf_nat]
[<ffffffffc077278e>] nf_nat_ipv4_fn+0x1be/0x230 [nf_nat_ipv4]
[<ffffffffc08a8020>] ? iptable_nat_ipv4_fn+0x20/0x20 [iptable_nat]
[<ffffffffc08a8018>] iptable_nat_ipv4_fn+0x18/0x20 [iptable_nat]
[<ffffffffaa08f418>] nf_iterate+0x98/0xe0
[<ffffffffaa08f508>] nf_hook_slow+0xa8/0x110
[<ffffffffaa099bc6>] ip_local_deliver+0xc6/0xe0
[<ffffffffaa0997b0>] ? ip_rcv_finish+0x370/0x370
[<ffffffffaa0994d0>] ip_rcv_finish+0x90/0x370
[<ffffffffaa099ea0>] ip_rcv+0x2c0/0x420
[<ffffffffaa099440>] ? inet_del_offload+0x40/0x40
[<ffffffffaa04ebd9>] __netif_receive_skb_core+0x729/0xa10
[<ffffffffa9b06e6f>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffffaa04eed8>] __netif_receive_skb+0x18/0x60
[<ffffffffaa04ef60>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffaa04fbe8>] napi_gro_receive+0xd8/0x100
[<ffffffffc03826b4>] vmxnet3_rq_rx_complete+0x924/0xef0 [vmxnet3]
[<ffffffffc03830e6>] vmxnet3_poll_rx_only+0x36/0xa0 [vmxnet3]
[<ffffffffaa04f57f>] net_rx_action+0x26f/0x390
[<ffffffffa9aa5305>] __do_softirq+0xf5/0x280
[<ffffffffa9aa54c8>] run_ksoftirqd+0x38/0x50
[<ffffffffa9acec24>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffa9aceae0>] ? lg_double_unlock+0x40/0x40
[<ffffffffa9ac61f1>] kthread+0xd1/0xe0
[<ffffffffa9ac6120>] ? insert_kthread_work+0x40/0x40
[<ffffffffaa18cd37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffa9ac6120>] ? insert_kthread_work+0x40/0x40
Code: d2 74 56 0f b7 42 12 66 85 c0 74 4d 48 01 d0 48 89 58 10 49 8b 96 28 0b 00 00 4a 8d 14 e2 48 8b 0a 48 89 50 08 48 89 08 48 85 c9 <48> 89 02 74 04 48 89 41 08 48 c7 c7 78 f4 89 c0 e8 c9 5e 8e e9
Complete report #21492
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
nf_nat_setup_info
|
nf_nat |
0xffffffffc08213d2
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x2e2
|
|
2
|
__nf_nat_alloc_null_binding
|
nf_nat |
0xffffffffc0821467
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x57
|
|
3
|
nf_nat_alloc_null_binding
|
nf_nat |
0xffffffffc08214b1
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x21
|
|
4
|
nf_nat_ipv4_fn
|
nf_nat_ipv4 |
0xffffffffc080378e
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x1be
|
|
5
|
|
iptable_nat |
0xffffffffc0828020
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x20
|
|
6
|
iptable_nat_ipv4_fn
|
iptable_nat |
0xffffffffc0828018
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x18
|
|
7
|
nf_iterate
|
vmlinux |
0xffffffffae690418
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x98
|
|
8
|
nf_hook_slow
|
vmlinux |
0xffffffffae690508
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0xa8
|
|
9
|
ip_local_deliver
|
vmlinux |
0xffffffffae69abc6
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0xc6
|
|
10
|
|
vmlinux |
0xffffffffae69a7b0
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x370
|
|
11
|
ip_rcv_finish
|
vmlinux |
0xffffffffae69a4d0
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x90
|
|
12
|
ip_rcv
|
vmlinux |
0xffffffffae69aea0
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x2c0
|
|
13
|
|
vmlinux |
0xffffffffae69a440
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x40
|
|
14
|
__netif_receive_skb_core
|
vmlinux |
0xffffffffae64fbd9
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x729
|
|
15
|
|
vmlinux |
0xffffffffae6c7600
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x110
|
|
16
|
|
vmlinux |
0xffffffffae1076bf
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x3f
|
|
17
|
__netif_receive_skb
|
vmlinux |
0xffffffffae64fed8
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x18
|
|
18
|
netif_receive_skb_internal
|
vmlinux |
0xffffffffae64ff60
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x40
|
|
19
|
napi_gro_receive
|
vmlinux |
0xffffffffae650be8
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0xd8
|
|
20
|
vmxnet3_rq_rx_complete
|
vmxnet3 |
0xffffffffc041e6b4
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x924
|
|
21
|
vmxnet3_poll_rx_only
|
vmxnet3 |
0xffffffffc041f0e6
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x36
|
|
22
|
net_rx_action
|
vmlinux |
0xffffffffae65057f
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x26f
|
|
23
|
__do_softirq
|
vmlinux |
0xffffffffae0a5305
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0xf5
|
|
24
|
run_ksoftirqd
|
vmlinux |
0xffffffffae0a54c8
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x38
|
|
25
|
smpboot_thread_fn
|
vmlinux |
0xffffffffae0cec24
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x144
|
|
26
|
|
vmlinux |
0xffffffffae0ceae0
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x40
|
|
27
|
kthread
|
vmlinux |
0xffffffffae0c61f1
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0xd1
|
|
28
|
|
vmlinux |
0xffffffffae0c6120
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x40
|
|
29
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffae78dd37
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x21
|
|
30
|
|
vmlinux |
0xffffffffae0c6120
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 24s! [ksoftirqd/0:6]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter tcp_diag inet_diag vmw_vsock_vmci_transport vsock sunrpc sb_edac coretemp iosf_mbi crc32_pclmul ppdev ghash_clmulni_intel vmw_balloon aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr sg joydev pcc_cpufreq parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic vmwgfx sr_mod cdrom drm_kms_helper ata_generic syscopyarea pata_acpi sysfillrect sysimgblt fb_sys_fops ttm drm mptspi ata_piix scsi_transport_spi mptscsih crct10dif_pclmul libata crct10dif_common
mptbase crc32c_intel serio_raw vmxnet3 drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 6 Comm: ksoftirqd/0 Kdump: loaded Tainted: G L ------------ 3.10.0-1062.9.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013
task: ffff9c95674e3150 ti: ffff9c95674f8000 task.ti: ffff9c95674f8000
RIP: 0010:[<ffffffffc08213d2>] [<ffffffffc08213d2>] nf_nat_setup_info+0x2e2/0x320 [nf_nat]
RSP: 0018:ffff9c95674fb930 EFLAGS: 00010246
RAX: ffff9c9535725958 RBX: ffff9c95357258e8 RCX: 0000000000000000
RDX: ffff9c960bff7288 RSI: 000000009e6f06e1 RDI: ffffffffc0824478
RBP: ffff9c95674fb9d0 R08: ffff9c95357258e8 R09: ffff9c95674fb930
R10: ffff9c95674fb8e0 R11: ffffffffc04cb5f8 R12: ffffffffaed13a40
R13: ffff9c95357258e8 R14: ffffffffc082101c R15: ffff9c95674fb920
FS: 0000000000000000(0000) GS:ffff9c9568800000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000000b673b000 CR3: 00000000348c6000 CR4: 00000000000407f0
Call Trace:
[<ffffffffc0821467>] __nf_nat_alloc_null_binding+0x57/0x80 [nf_nat]
[<ffffffffc08214b1>] nf_nat_alloc_null_binding+0x21/0x30 [nf_nat]
[<ffffffffc080378e>] nf_nat_ipv4_fn+0x1be/0x230 [nf_nat_ipv4]
[<ffffffffc0828020>] ? iptable_nat_ipv4_fn+0x20/0x20 [iptable_nat]
[<ffffffffc0828018>] iptable_nat_ipv4_fn+0x18/0x20 [iptable_nat]
[<ffffffffae690418>] nf_iterate+0x98/0xe0
[<ffffffffae690508>] nf_hook_slow+0xa8/0x110
[<ffffffffae69abc6>] ip_local_deliver+0xc6/0xe0
[<ffffffffae69a7b0>] ? ip_rcv_finish+0x370/0x370
[<ffffffffae69a4d0>] ip_rcv_finish+0x90/0x370
[<ffffffffae69aea0>] ip_rcv+0x2c0/0x420
[<ffffffffae69a440>] ? inet_del_offload+0x40/0x40
[<ffffffffae64fbd9>] __netif_receive_skb_core+0x729/0xa10
[<ffffffffae6c7600>] ? tcp4_gro_receive+0x110/0x1a0
[<ffffffffae1076bf>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffffae64fed8>] __netif_receive_skb+0x18/0x60
[<ffffffffae64ff60>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffae650be8>] napi_gro_receive+0xd8/0x100
[<ffffffffc041e6b4>] vmxnet3_rq_rx_complete+0x924/0xef0 [vmxnet3]
[<ffffffffc041f0e6>] vmxnet3_poll_rx_only+0x36/0xa0 [vmxnet3]
[<ffffffffae65057f>] net_rx_action+0x26f/0x390
[<ffffffffae0a5305>] __do_softirq+0xf5/0x280
[<ffffffffae0a54c8>] run_ksoftirqd+0x38/0x50
[<ffffffffae0cec24>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffae0ceae0>] ? lg_double_unlock+0x40/0x40
[<ffffffffae0c61f1>] kthread+0xd1/0xe0
[<ffffffffae0c6120>] ? insert_kthread_work+0x40/0x40
[<ffffffffae78dd37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffae0c6120>] ? insert_kthread_work+0x40/0x40
Code: d2 74 56 0f b7 42 12 66 85 c0 74 4d 48 01 d0 48 89 58 10 49 8b 96 28 0b 00 00 4a 8d 14 e2 48 8b 0a 48 89 50 08 48 89 08 48 85 c9 <48> 89 02 74 04 48 89 41 08 48 c7 c7 78 44 82 c0 e8 c9 1d f6 ed