Info

Permalink
Function
nf_nat_setup_info
First occurrence
2020-12-14
Last occurrence
2026-11-03
State
NEW
Components
kernel
Quality
-63

Statistics

Operating system Count
CentOS 7 13
Architecture Count
x86_64 13
Related packages Count
kernel 13
 0:3.10.0-1160.119.1.el7 3
 0:3.10.0-1062.9.1.el7 2
 0:3.10.0-957.12.2.el7 2
 0:3.10.0-1062.4.1.el7 1
 0:3.10.0-862.14.4.el7 1
 0:3.10.0-1062.4.3.el7 1
 0:3.10.0-957.el7 1
 0:3.10.0-1160.el7 1
 0:3.10.0-1127.18.2.el7 1

Packages names with count less than the total count of reports are most probably not the cause of the problem.

History

Daily:

Weekly:

Monthly:

Report backtrace


Complete report #1261396
Frame # Function Binary Source or offset Line
1
nf_nat_setup_info
nf_nat 0xffffffffc07223d2
Build id: 3.10.0-1127.18.2.el7.x86_64
0x2e2
2
__nf_nat_alloc_null_binding
nf_nat 0xffffffffc0722467
Build id: 3.10.0-1127.18.2.el7.x86_64
0x57
3
nf_nat_alloc_null_binding
nf_nat 0xffffffffc07224b1
Build id: 3.10.0-1127.18.2.el7.x86_64
0x21
4
nf_nat_ipv4_fn
nf_nat_ipv4 0xffffffffc071778e
Build id: 3.10.0-1127.18.2.el7.x86_64
0x1be
5
? iptable_nat_ipv4_fn
iptable_nat 0xffffffffc0729020
Build id: 3.10.0-1127.18.2.el7.x86_64
0x20
6
iptable_nat_ipv4_fn
iptable_nat 0xffffffffc0729018
Build id: 3.10.0-1127.18.2.el7.x86_64
0x18
7
nf_iterate
vmlinux 0xffffffff93693cf8
Build id: 3.10.0-1127.18.2.el7.x86_64
0x98
8
nf_hook_slow
vmlinux 0xffffffff93693de8
Build id: 3.10.0-1127.18.2.el7.x86_64
0xa8
9
ip_local_deliver
vmlinux 0xffffffff9369e456
Build id: 3.10.0-1127.18.2.el7.x86_64
0xc6
10
? ip_rcv_finish
vmlinux 0xffffffff9369e040
Build id: 3.10.0-1127.18.2.el7.x86_64
0x370
11
ip_rcv_finish
vmlinux 0xffffffff9369dd60
Build id: 3.10.0-1127.18.2.el7.x86_64
0x90
12
ip_rcv
vmlinux 0xffffffff9369e730
Build id: 3.10.0-1127.18.2.el7.x86_64
0x2c0
13
? inet_del_offload
vmlinux 0xffffffff9369dcd0
Build id: 3.10.0-1127.18.2.el7.x86_64
0x40
14
__netif_receive_skb_core
vmlinux 0xffffffff93653c19
Build id: 3.10.0-1127.18.2.el7.x86_64
0x729
15
? tcp4_gro_receive
vmlinux 0xffffffff936caf00
Build id: 3.10.0-1127.18.2.el7.x86_64
0xa0
16
? __getnstimeofday64
vmlinux 0xffffffff931082af
Build id: 3.10.0-1127.18.2.el7.x86_64
0x3f
17
__netif_receive_skb
vmlinux 0xffffffff93653f18
Build id: 3.10.0-1127.18.2.el7.x86_64
0x18
18
netif_receive_skb_internal
vmlinux 0xffffffff93653fa0
Build id: 3.10.0-1127.18.2.el7.x86_64
0x40
19
napi_gro_receive
vmlinux 0xffffffff93654c28
Build id: 3.10.0-1127.18.2.el7.x86_64
0xd8
20
vmxnet3_rq_rx_complete
vmxnet3 0xffffffffc02486b4
Build id: 3.10.0-1127.18.2.el7.x86_64
0x924
21
vmxnet3_poll_rx_only
vmxnet3 0xffffffffc02490e6
Build id: 3.10.0-1127.18.2.el7.x86_64
0x36
22
net_rx_action
vmlinux 0xffffffff936545bf
Build id: 3.10.0-1127.18.2.el7.x86_64
0x26f
23
__do_softirq
vmlinux 0xffffffff930a5695
Build id: 3.10.0-1127.18.2.el7.x86_64
0xf5
24
run_ksoftirqd
vmlinux 0xffffffff930a5858
Build id: 3.10.0-1127.18.2.el7.x86_64
0x38
25
smpboot_thread_fn
vmlinux 0xffffffff930cf0e4
Build id: 3.10.0-1127.18.2.el7.x86_64
0x144
26
? lg_double_unlock
vmlinux 0xffffffff930cefa0
Build id: 3.10.0-1127.18.2.el7.x86_64
0x40
27
kthread
vmlinux 0xffffffff930c6691
Build id: 3.10.0-1127.18.2.el7.x86_64
0xd1
28
? insert_kthread_work
vmlinux 0xffffffff930c65c0
Build id: 3.10.0-1127.18.2.el7.x86_64
0x40
29
ret_from_fork_nospec_begin
vmlinux 0xffffffff93792d37
Build id: 3.10.0-1127.18.2.el7.x86_64
0x21
30
? insert_kthread_work
vmlinux 0xffffffff930c65c0
Build id: 3.10.0-1127.18.2.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 26s! [ksoftirqd/2:19]
Modules linked in: falcon_lsm_serviceable(PE) falcon_nf_netcontain(E) falcon_kal(E) falcon_lsm_pinned_17804(E) xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter devlink ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw ppdev gf128mul glue_helper ablk_helper vmw_balloon cryptd pcspkr sg joydev parport_pc i2c_piix4 vmw_vmci parport sch_fq_codel ip_tables ext4 mbcache jbd2 ata_generic pata_acpi vmwgfx drm_kms_helper sd_mod syscopyarea crc_t10dif sysfillrect crct10dif_generic
sysimgblt fb_sys_fops ttm nfit ahci drm libnvdimm ata_piix libahci crct10dif_pclmul crct10dif_common crc32c_intel libata serio_raw vmxnet3 vmw_pvscsi drm_panel_orientation_quirks
CPU: 2 PID: 19 Comm: ksoftirqd/2 Kdump: loaded Tainted: P            EL ------------   3.10.0-1127.18.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff9021b9c1a0e0 ti: ffff9021b9c28000 task.ti: ffff9021b9c28000
RIP: 0010:[<ffffffffc07223d2>]  [<ffffffffc07223d2>] nf_nat_setup_info+0x2e2/0x320 [nf_nat]
RSP: 0018:ffff9021b9c2b930  EFLAGS: 00010246
RAX: ffff90246dc15c98 RBX: ffff90246dc15c28 RCX: 0000000000000000
RDX: ffff9020f6db7dc8 RSI: 000000003e0085d6 RDI: ffffffffc0725478
RBP: ffff9021b9c2b9d0 R08: ffff90246dc15c28 R09: ffff9021b9c2b930
R10: ffff9021b9c2b8e0 R11: ffffffffc03ff5f8 R12: ffffffff93d15c80
R13: ffff90246dc15c28 R14: ffffffffc072201c R15: ffff9021b9c2b920
FS:  0000000000000000(0000) GS:ffff90247fc80000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ff3f2f01000 CR3: 0000000034dc2000 CR4: 00000000000607e0
Call Trace:
[<ffffffffc0722467>] __nf_nat_alloc_null_binding+0x57/0x80 [nf_nat]
[<ffffffffc07224b1>] nf_nat_alloc_null_binding+0x21/0x30 [nf_nat]
[<ffffffffc071778e>] nf_nat_ipv4_fn+0x1be/0x230 [nf_nat_ipv4]
[<ffffffffc0729020>] ? iptable_nat_ipv4_fn+0x20/0x20 [iptable_nat]
[<ffffffffc0729018>] iptable_nat_ipv4_fn+0x18/0x20 [iptable_nat]
[<ffffffff93693cf8>] nf_iterate+0x98/0xe0
[<ffffffff93693de8>] nf_hook_slow+0xa8/0x110
[<ffffffff9369e456>] ip_local_deliver+0xc6/0xe0
[<ffffffff9369e040>] ? ip_rcv_finish+0x370/0x370
[<ffffffff9369dd60>] ip_rcv_finish+0x90/0x370
[<ffffffff9369e730>] ip_rcv+0x2c0/0x420
[<ffffffff9369dcd0>] ? inet_del_offload+0x40/0x40
[<ffffffff93653c19>] __netif_receive_skb_core+0x729/0xa10
[<ffffffff936caf00>] ? tcp4_gro_receive+0xa0/0x1a0
[<ffffffff931082af>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffff93653f18>] __netif_receive_skb+0x18/0x60
[<ffffffff93653fa0>] netif_receive_skb_internal+0x40/0xc0
[<ffffffff93654c28>] napi_gro_receive+0xd8/0x100
[<ffffffffc02486b4>] vmxnet3_rq_rx_complete+0x924/0xef0 [vmxnet3]
[<ffffffffc02490e6>] vmxnet3_poll_rx_only+0x36/0xa0 [vmxnet3]
[<ffffffff936545bf>] net_rx_action+0x26f/0x390
[<ffffffff930a5695>] __do_softirq+0xf5/0x280
[<ffffffff930a5858>] run_ksoftirqd+0x38/0x50
[<ffffffff930cf0e4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff930cefa0>] ? lg_double_unlock+0x40/0x40
[<ffffffff930c6691>] kthread+0xd1/0xe0
[<ffffffff930c65c0>] ? insert_kthread_work+0x40/0x40
[<ffffffff93792d37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff930c65c0>] ? insert_kthread_work+0x40/0x40
Code: d2 74 56 0f b7 42 12 66 85 c0 74 4d 48 01 d0 48 89 58 10 49 8b 96 28 0b 00 00 4a 8d 14 e2 48 8b 0a 48 89 50 08 48 89 08 48 85 c9 <48> 89 02 74 04 48 89 41 08 48 c7 c7 78 54 72 c0 e8 69 61 06 d3 
Complete report #1102052
Frame # Function Binary Source or offset Line
1
nf_nat_setup_info
nf_nat 0xffffffffc07473c5
Build id: 3.10.0-1160.el7.x86_64
0x2d5
2
__nf_nat_alloc_null_binding
nf_nat 0xffffffffc0747467
Build id: 3.10.0-1160.el7.x86_64
0x57
3
nf_nat_alloc_null_binding
nf_nat 0xffffffffc07474b1
Build id: 3.10.0-1160.el7.x86_64
0x21
4
nf_nat_ipv4_fn
nf_nat_ipv4 0xffffffffc073c78e
Build id: 3.10.0-1160.el7.x86_64
0x1be
5
? iptable_nat_ipv4_fn
iptable_nat 0xffffffffc074e020
Build id: 3.10.0-1160.el7.x86_64
0x20
6
iptable_nat_ipv4_fn
iptable_nat 0xffffffffc074e018
Build id: 3.10.0-1160.el7.x86_64
0x18
7
nf_iterate
vmlinux 0xffffffffbac94e38
Build id: 3.10.0-1160.el7.x86_64
0x98
8
nf_hook_slow
vmlinux 0xffffffffbac94f28
Build id: 3.10.0-1160.el7.x86_64
0xa8
9
ip_local_deliver
vmlinux 0xffffffffbac9f5e6
Build id: 3.10.0-1160.el7.x86_64
0xc6
10
? ip_rcv_finish
vmlinux 0xffffffffbac9f1d0
Build id: 3.10.0-1160.el7.x86_64
0x370
11
ip_rcv_finish
vmlinux 0xffffffffbac9eef0
Build id: 3.10.0-1160.el7.x86_64
0x90
12
ip_rcv
vmlinux 0xffffffffbac9f8c0
Build id: 3.10.0-1160.el7.x86_64
0x2c0
13
? inet_del_offload
vmlinux 0xffffffffbac9ee60
Build id: 3.10.0-1160.el7.x86_64
0x40
14
__netif_receive_skb_core
vmlinux 0xffffffffbac54c19
Build id: 3.10.0-1160.el7.x86_64
0x729
15
? tcp4_gro_receive
vmlinux 0xffffffffbaccc100
Build id: 3.10.0-1160.el7.x86_64
0x80
16
? inet_gro_receive
vmlinux 0xffffffffbacddd99
Build id: 3.10.0-1160.el7.x86_64
0x249
17
__netif_receive_skb
vmlinux 0xffffffffbac54f18
Build id: 3.10.0-1160.el7.x86_64
0x18
18
netif_receive_skb_internal
vmlinux 0xffffffffbac54fa0
Build id: 3.10.0-1160.el7.x86_64
0x40
19
napi_gro_receive
vmlinux 0xffffffffbac55c28
Build id: 3.10.0-1160.el7.x86_64
0xd8
20
e1000_clean_rx_irq
e1000 0xffffffffc024545b
Build id: 3.10.0-1160.el7.x86_64
0x2bb
21
e1000_clean
e1000 0xffffffffc0246100
Build id: 3.10.0-1160.el7.x86_64
0x280
22
? update_curr
vmlinux 0xffffffffba6e43cc
Build id: 3.10.0-1160.el7.x86_64
0x14c
23
net_rx_action
vmlinux 0xffffffffbac555bf
Build id: 3.10.0-1160.el7.x86_64
0x26f
24
__do_softirq
vmlinux 0xffffffffba6a4b95
Build id: 3.10.0-1160.el7.x86_64
0xf5
25
run_ksoftirqd
vmlinux 0xffffffffba6a4d58
Build id: 3.10.0-1160.el7.x86_64
0x38
26
smpboot_thread_fn
vmlinux 0xffffffffba6ce674
Build id: 3.10.0-1160.el7.x86_64
0x144
27
? lg_double_unlock
vmlinux 0xffffffffba6ce530
Build id: 3.10.0-1160.el7.x86_64
0x40
28
kthread
vmlinux 0xffffffffba6c5c21
Build id: 3.10.0-1160.el7.x86_64
0xd1
29
? insert_kthread_work
vmlinux 0xffffffffba6c5b50
Build id: 3.10.0-1160.el7.x86_64
0x40
30
ret_from_fork_nospec_begin
vmlinux 0xffffffffbad93df7
Build id: 3.10.0-1160.el7.x86_64
0x21
31
? insert_kthread_work
vmlinux 0xffffffffba6c5b50
Build id: 3.10.0-1160.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [ksoftirqd/0:6]
Modules linked in: tcp_lp xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr joydev ppdev vmw_balloon sg parport_pc parport vmw_vmci i2c_piix4 ip_tables ext4 mbcache jbd2 sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm nfit ata_piix mptspi drm scsi_transport_spi libnvdimm mptscsih libata mptbase
crct10dif_pclmul crct10dif_common crc32c_intel e1000 serio_raw drm_panel_orientation_quirks fuse
CPU: 0 PID: 6 Comm: ksoftirqd/0 Tainted: G             L ------------   3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff8c56bd765280 ti: ffff8c56be364000 task.ti: ffff8c56be364000
RIP: 0010:[<ffffffffc07473c5>]  [<ffffffffc07473c5>] nf_nat_setup_info+0x2d5/0x320 [nf_nat]
RSP: 0018:ffff8c56be367890  EFLAGS: 00010286
RAX: ffff8c56b57fbd18 RBX: ffff8c56b57fbca8 RCX: 00000000b969b2c6
RDX: ffff8c55f6b7a730 RSI: 000000007fd1b35c RDI: ffffffffc074a478
RBP: ffff8c56be367930 R08: ffff8c56b57fbca8 R09: ffff8c56be367890
R10: ffff8c56be367840 R11: ffffffffc04305f8 R12: ffffffffbb315e40
R13: ffff8c56b57fbca8 R14: ffffffffc074701c R15: ffff8c56be367880
FS:  0000000000000000(0000) GS:ffff8c5775600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fe28bc2a480 CR3: 0000000233496000 CR4: 0000000000360ff0
Call Trace:
[<ffffffffc0747467>] __nf_nat_alloc_null_binding+0x57/0x80 [nf_nat]
[<ffffffffc07474b1>] nf_nat_alloc_null_binding+0x21/0x30 [nf_nat]
[<ffffffffc073c78e>] nf_nat_ipv4_fn+0x1be/0x230 [nf_nat_ipv4]
[<ffffffffc074e020>] ? iptable_nat_ipv4_fn+0x20/0x20 [iptable_nat]
[<ffffffffc074e018>] iptable_nat_ipv4_fn+0x18/0x20 [iptable_nat]
[<ffffffffbac94e38>] nf_iterate+0x98/0xe0
[<ffffffffbac94f28>] nf_hook_slow+0xa8/0x110
[<ffffffffbac9f5e6>] ip_local_deliver+0xc6/0xe0
[<ffffffffbac9f1d0>] ? ip_rcv_finish+0x370/0x370
[<ffffffffbac9eef0>] ip_rcv_finish+0x90/0x370
[<ffffffffbac9f8c0>] ip_rcv+0x2c0/0x420
[<ffffffffbac9ee60>] ? inet_del_offload+0x40/0x40
[<ffffffffbac54c19>] __netif_receive_skb_core+0x729/0xa10
[<ffffffffbaccc100>] ? tcp4_gro_receive+0x80/0x1a0
[<ffffffffbacddd99>] ? inet_gro_receive+0x249/0x290
[<ffffffffbac54f18>] __netif_receive_skb+0x18/0x60
[<ffffffffbac54fa0>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffbac55c28>] napi_gro_receive+0xd8/0x100
[<ffffffffc024545b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc0246100>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffffba6e43cc>] ? update_curr+0x14c/0x1e0
[<ffffffffbac555bf>] net_rx_action+0x26f/0x390
[<ffffffffba6a4b95>] __do_softirq+0xf5/0x280
[<ffffffffba6a4d58>] run_ksoftirqd+0x38/0x50
[<ffffffffba6ce674>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffba6ce530>] ? lg_double_unlock+0x40/0x40
[<ffffffffba6c5c21>] kthread+0xd1/0xe0
[<ffffffffba6c5b50>] ? insert_kthread_work+0x40/0x40
[<ffffffffbad93df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffba6c5b50>] ? insert_kthread_work+0x40/0x40
Code: 48 8b 93 e8 00 00 00 49 c1 ec 20 48 85 d2 74 56 0f b7 42 12 66 85 c0 74 4d 48 01 d0 48 89 58 10 49 8b 96 28 0b 00 00 4a 8d 14 e2 <48> 8b 0a 48 89 50 08 48 89 08 48 85 c9 48 89 02 74 04 48 89 41 
Complete report #704392
Frame # Function Binary Source or offset Line
1
native_queued_spin_lock_slowpath
vmlinux 0xffffffff8d711ffd
Build id: 3.10.0-957.el7.x86_64
0x1d
2
queued_spin_lock_slowpath
vmlinux 0xffffffff8dd5bf4b
Build id: 3.10.0-957.el7.x86_64
0xb
3
_raw_spin_lock_bh
vmlinux 0xffffffff8dd6a443
Build id: 3.10.0-957.el7.x86_64
0x33
4
nf_nat_setup_info
nf_nat 0xffffffffc064d36a
Build id: 3.10.0-957.el7.x86_64
0x2aa
5
__nf_nat_alloc_null_binding
nf_nat 0xffffffffc064d437
Build id: 3.10.0-957.el7.x86_64
0x57
6
nf_nat_alloc_null_binding
nf_nat 0xffffffffc064d481
Build id: 3.10.0-957.el7.x86_64
0x21
7
nf_nat_ipv4_fn
nf_nat_ipv4 0xffffffffc065978e
Build id: 3.10.0-957.el7.x86_64
0x1be
8
? iptable_nat_ipv4_fn
iptable_nat 0xffffffffc0654020
Build id: 3.10.0-957.el7.x86_64
0x20
9
iptable_nat_ipv4_fn
iptable_nat 0xffffffffc0654018
Build id: 3.10.0-957.el7.x86_64
0x18
10
nf_iterate
vmlinux 0xffffffff8dc784a8
Build id: 3.10.0-957.el7.x86_64
0x98
11
nf_hook_slow
vmlinux 0xffffffff8dc78598
Build id: 3.10.0-957.el7.x86_64
0xa8
12
ip_local_deliver
vmlinux 0xffffffff8dc82ba2
Build id: 3.10.0-957.el7.x86_64
0xb2
13
? ip_rcv_finish
vmlinux 0xffffffff8dc827a0
Build id: 3.10.0-957.el7.x86_64
0x370
14
ip_rcv_finish
vmlinux 0xffffffff8dc824c0
Build id: 3.10.0-957.el7.x86_64
0x90
15
ip_rcv
vmlinux 0xffffffff8dc82e79
Build id: 3.10.0-957.el7.x86_64
0x2b9
16
? inet_del_offload
vmlinux 0xffffffff8dc82430
Build id: 3.10.0-957.el7.x86_64
0x40
17
__netif_receive_skb_core
vmlinux 0xffffffff8dc394f9
Build id: 3.10.0-957.el7.x86_64
0x729
18
__netif_receive_skb
vmlinux 0xffffffff8dc397f8
Build id: 3.10.0-957.el7.x86_64
0x18
19
process_backlog
vmlinux 0xffffffff8dc3a7be
Build id: 3.10.0-957.el7.x86_64
0xae
20
net_rx_action
vmlinux 0xffffffff8dc39e9f
Build id: 3.10.0-957.el7.x86_64
0x26f
21
__do_softirq
vmlinux 0xffffffff8d6a0f05
Build id: 3.10.0-957.el7.x86_64
0xf5
22
run_ksoftirqd
vmlinux 0xffffffff8d6a10c8
Build id: 3.10.0-957.el7.x86_64
0x38
23
smpboot_thread_fn
vmlinux 0xffffffff8d6ca5c4
Build id: 3.10.0-957.el7.x86_64
0x144
24
? lg_double_unlock
vmlinux 0xffffffff8d6ca480
Build id: 3.10.0-957.el7.x86_64
0x40
25
kthread
vmlinux 0xffffffff8d6c1c31
Build id: 3.10.0-957.el7.x86_64
0xd1
26
? insert_kthread_work
vmlinux 0xffffffff8d6c1b60
Build id: 3.10.0-957.el7.x86_64
0x40
27
ret_from_fork_nospec_begin
vmlinux 0xffffffff8dd74c37
Build id: 3.10.0-957.el7.x86_64
0x21
28
? insert_kthread_work
vmlinux 0xffffffff8d6c1b60
Build id: 3.10.0-957.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#11 stuck for 22s! [xfsaild/dm-2:9475]
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 26s! [ksoftirqd/0:3]
Modules linked in: binfmt_misc veth xt_nat xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock fuse vmhgfs(OE) overlay(T) sunrpc snd_seq_midi snd_seq_midi_event sb_edac coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel snd_ens1371 lrw snd_rawmidi gf128mul ppdev glue_helper snd_ac97_codec ablk_helper cryptd ac97_bus vmw_balloon snd_seq snd_seq_device snd_pcm snd_timer snd joydev soundcore pcspkr sg vmw_vmci i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sd_mod sr_mod cdrom crc_t10dif crct10dif_generic
ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm libata mptspi scsi_transport_spi mptscsih e1000 mptbase nfit libnvdimm crct10dif_pclmul crct10dif_common crc32c_intel serio_raw drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 3 Comm: ksoftirqd/0 Kdump: loaded Tainted: G           OEL ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
task: ffff8ce6fa35a080 ti: ffff8ce6f9c58000 task.ti: ffff8ce6f9c58000
RIP: 0010:[<ffffffff8d711ffd>]  [<ffffffff8d711ffd>] native_queued_spin_lock_slowpath+0x1d/0x200
RSP: 0018:ffff8ce6f9c5b9e0  EFLAGS: 00000293
RAX: 0000000000000001 RBX: ffff8ce596ed17c0 RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffffffffc0650478
RBP: ffff8ce6f9c5b9e0 R08: ffff8ce8997a0928 R09: ffff8ce6f9c5ba10
R10: ffff8ce6f9c5b9c0 R11: ffffffffc016f5f8 R12: 0000000000000000
R13: ffff8ce6f9c5bac0 R14: ffff8ce6f9c5ba38 R15: 00010000f9c5b9a8
FS:  0000000000000000(0000) GS:ffff8ce9ad600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f388e269120 CR3: 00000002a7604000 CR4: 00000000003607f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff8dd5bf4b>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffff8dd6a443>] _raw_spin_lock_bh+0x33/0x40
[<ffffffffc064d36a>] nf_nat_setup_info+0x2aa/0x320 [nf_nat]
[<ffffffffc064d437>] __nf_nat_alloc_null_binding+0x57/0x80 [nf_nat]
[<ffffffffc064d481>] nf_nat_alloc_null_binding+0x21/0x30 [nf_nat]
[<ffffffffc065978e>] nf_nat_ipv4_fn+0x1be/0x230 [nf_nat_ipv4]
[<ffffffffc0654020>] ? iptable_nat_ipv4_fn+0x20/0x20 [iptable_nat]
[<ffffffffc0654018>] iptable_nat_ipv4_fn+0x18/0x20 [iptable_nat]
[<ffffffff8dc784a8>] nf_iterate+0x98/0xe0
[<ffffffff8dc78598>] nf_hook_slow+0xa8/0x110
[<ffffffff8dc82ba2>] ip_local_deliver+0xb2/0xd0
[<ffffffff8dc827a0>] ? ip_rcv_finish+0x370/0x370
[<ffffffff8dc824c0>] ip_rcv_finish+0x90/0x370
[<ffffffff8dc82e79>] ip_rcv+0x2b9/0x410
[<ffffffff8dc82430>] ? inet_del_offload+0x40/0x40
[<ffffffff8dc394f9>] __netif_receive_skb_core+0x729/0xa10
[<ffffffff8dc397f8>] __netif_receive_skb+0x18/0x60
[<ffffffff8dc3a7be>] process_backlog+0xae/0x180
[<ffffffff8dc39e9f>] net_rx_action+0x26f/0x390
[<ffffffff8d6a0f05>] __do_softirq+0xf5/0x280
[<ffffffff8d6a10c8>] run_ksoftirqd+0x38/0x50
[<ffffffff8d6ca5c4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff8d6ca480>] ? lg_double_unlock+0x40/0x40
[<ffffffff8d6c1c31>] kthread+0xd1/0xe0
[<ffffffff8d6c1b60>] ? insert_kthread_work+0x40/0x40
[<ffffffff8dd74c37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff8d6c1b60>] ? insert_kthread_work+0x40/0x40
Code: 47 fe ff ff 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 66 90 b9 01 00 00 00 8b 17 85 d2 74 0d 83 fa 03 74 08 f3 90 <8b> 17 85 d2 75 f3 89 d0 f0 0f b1 0f 39 c2 75 e3 5d 66 90 c3 0f 
Complete report #491612
Frame # Function Binary Source or offset Line
1
native_queued_spin_lock_slowpath
vmlinux 0xffffffffa21122cd
Build id: 3.10.0-957.12.2.el7.x86_64
0x1d
2
queued_spin_lock_slowpath
vmlinux 0xffffffffa275d1cb
Build id: 3.10.0-957.12.2.el7.x86_64
0xb
3
_raw_spin_lock_bh
vmlinux 0xffffffffa276b6e3
Build id: 3.10.0-957.12.2.el7.x86_64
0x33
4
nf_nat_setup_info
nf_nat 0xffffffffc059a36a
Build id: 3.10.0-957.12.2.el7.x86_64
0x2aa
5
__nf_nat_alloc_null_binding
nf_nat 0xffffffffc059a437
Build id: 3.10.0-957.12.2.el7.x86_64
0x57
6
nf_nat_alloc_null_binding
nf_nat 0xffffffffc059a481
Build id: 3.10.0-957.12.2.el7.x86_64
0x21
7
nf_nat_ipv4_fn
nf_nat_ipv4 0xffffffffc05ed78e
Build id: 3.10.0-957.12.2.el7.x86_64
0x1be
8
? iptable_nat_ipv4_fn
iptable_nat 0xffffffffc05e8020
Build id: 3.10.0-957.12.2.el7.x86_64
0x20
9
iptable_nat_ipv4_fn
iptable_nat 0xffffffffc05e8018
Build id: 3.10.0-957.12.2.el7.x86_64
0x18
10
nf_iterate
vmlinux 0xffffffffa2679528
Build id: 3.10.0-957.12.2.el7.x86_64
0x98
11
nf_hook_slow
vmlinux 0xffffffffa2679618
Build id: 3.10.0-957.12.2.el7.x86_64
0xa8
12
ip_local_deliver
vmlinux 0xffffffffa2683c22
Build id: 3.10.0-957.12.2.el7.x86_64
0xb2
13
? ip_rcv_finish
vmlinux 0xffffffffa2683820
Build id: 3.10.0-957.12.2.el7.x86_64
0x370
14
ip_rcv_finish
vmlinux 0xffffffffa2683540
Build id: 3.10.0-957.12.2.el7.x86_64
0x90
15
ip_rcv
vmlinux 0xffffffffa2683ef9
Build id: 3.10.0-957.12.2.el7.x86_64
0x2b9
16
? inet_del_offload
vmlinux 0xffffffffa26834b0
Build id: 3.10.0-957.12.2.el7.x86_64
0x40
17
__netif_receive_skb_core
vmlinux 0xffffffffa263a579
Build id: 3.10.0-957.12.2.el7.x86_64
0x729
18
? tcp4_gro_receive
vmlinux 0xffffffffa26aff00
Build id: 3.10.0-957.12.2.el7.x86_64
0x100
19
? inet_gro_receive
vmlinux 0xffffffffa26c1889
Build id: 3.10.0-957.12.2.el7.x86_64
0x249
20
__netif_receive_skb
vmlinux 0xffffffffa263a878
Build id: 3.10.0-957.12.2.el7.x86_64
0x18
21
netif_receive_skb_internal
vmlinux 0xffffffffa263a900
Build id: 3.10.0-957.12.2.el7.x86_64
0x40
22
napi_gro_receive
vmlinux 0xffffffffa263b588
Build id: 3.10.0-957.12.2.el7.x86_64
0xd8
23
vmxnet3_rq_rx_complete
vmxnet3 0xffffffffc00d89cc
Build id: 3.10.0-957.12.2.el7.x86_64
0x91c
24
vmxnet3_poll_rx_only
vmxnet3 0xffffffffc00d90f6
Build id: 3.10.0-957.12.2.el7.x86_64
0x36
25
net_rx_action
vmlinux 0xffffffffa263af1f
Build id: 3.10.0-957.12.2.el7.x86_64
0x26f
26
__do_softirq
vmlinux 0xffffffffa20a0ff5
Build id: 3.10.0-957.12.2.el7.x86_64
0xf5
27
run_ksoftirqd
vmlinux 0xffffffffa20a11b8
Build id: 3.10.0-957.12.2.el7.x86_64
0x38
28
smpboot_thread_fn
vmlinux 0xffffffffa20ca6b4
Build id: 3.10.0-957.12.2.el7.x86_64
0x144
29
? lg_double_unlock
vmlinux 0xffffffffa20ca570
Build id: 3.10.0-957.12.2.el7.x86_64
0x40
30
kthread
vmlinux 0xffffffffa20c1d21
Build id: 3.10.0-957.12.2.el7.x86_64
0xd1
31
? insert_kthread_work
vmlinux 0xffffffffa20c1c50
Build id: 3.10.0-957.12.2.el7.x86_64
0x40
32
ret_from_fork_nospec_begin
vmlinux 0xffffffffa2775c37
Build id: 3.10.0-957.12.2.el7.x86_64
0x21
33
? insert_kthread_work
vmlinux 0xffffffffa20c1c50
Build id: 3.10.0-957.12.2.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 23s! [ksoftirqd/2:19]
Modules linked in: nfsv3 rpcsec_gss_krb5 nfsv4 nfs fscache cmac arc4 md4 nls_utf8 cifs ccm dns_resolver fuse xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables vmw_vsock_vmci_transport vsock iptable_filter dm_mirror dm_region_hash dm_log dm_mod sb_edac coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev vmw_balloon joydev pcspkr sg parport_pc parport vmw_vmci i2c_piix4 nfsd auth_rpcgss nfs_acl lockd grace sunrpc sch_fq_codel ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi crct10dif_pclmul
crct10dif_common crc32c_intel vmwgfx drm_kms_helper serio_raw syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm vmxnet3 mptspi libata scsi_transport_spi mptscsih mptbase drm_panel_orientation_quirks floppy
CPU: 2 PID: 19 Comm: ksoftirqd/2 Tainted: G        W    L ------------   3.10.0-957.12.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff8abadb264100 ti: ffff8abadb29c000 task.ti: ffff8abadb29c000
RIP: 0010:[<ffffffffa21122cd>]  [<ffffffffa21122cd>] native_queued_spin_lock_slowpath+0x1d/0x200
RSP: 0018:ffff8abadb29f910  EFLAGS: 00000293
RAX: 0000000000000001 RBX: ffff8abc4baa6800 RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffffffffc059d478
RBP: ffff8abadb29f910 R08: ffff8abef4fbae68 R09: ffff8abadb29f940
R10: ffff8abadb29f8f0 R11: ffffffffc02755f8 R12: 0000000000000000
R13: ffff8abadb29f9f0 R14: ffff8abadb29f968 R15: 00010000db29f8d8
FS:  0000000000000000(0000) GS:ffff8ad15e680000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000056415be7d000 CR3: 000000093c354000 CR4: 00000000001607e0
Call Trace:
[<ffffffffa275d1cb>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffffa276b6e3>] _raw_spin_lock_bh+0x33/0x40
[<ffffffffc059a36a>] nf_nat_setup_info+0x2aa/0x320 [nf_nat]
[<ffffffffc059a437>] __nf_nat_alloc_null_binding+0x57/0x80 [nf_nat]
[<ffffffffc059a481>] nf_nat_alloc_null_binding+0x21/0x30 [nf_nat]
[<ffffffffc05ed78e>] nf_nat_ipv4_fn+0x1be/0x230 [nf_nat_ipv4]
[<ffffffffc05e8020>] ? iptable_nat_ipv4_fn+0x20/0x20 [iptable_nat]
[<ffffffffc05e8018>] iptable_nat_ipv4_fn+0x18/0x20 [iptable_nat]
[<ffffffffa2679528>] nf_iterate+0x98/0xe0
[<ffffffffa2679618>] nf_hook_slow+0xa8/0x110
[<ffffffffa2683c22>] ip_local_deliver+0xb2/0xd0
[<ffffffffa2683820>] ? ip_rcv_finish+0x370/0x370
[<ffffffffa2683540>] ip_rcv_finish+0x90/0x370
[<ffffffffa2683ef9>] ip_rcv+0x2b9/0x410
[<ffffffffa26834b0>] ? inet_del_offload+0x40/0x40
[<ffffffffa263a579>] __netif_receive_skb_core+0x729/0xa10
[<ffffffffa26aff00>] ? tcp4_gro_receive+0x100/0x1a0
[<ffffffffa26c1889>] ? inet_gro_receive+0x249/0x290
[<ffffffffa263a878>] __netif_receive_skb+0x18/0x60
[<ffffffffa263a900>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffa263b588>] napi_gro_receive+0xd8/0x100
[<ffffffffc00d89cc>] vmxnet3_rq_rx_complete+0x91c/0xef0 [vmxnet3]
[<ffffffffc00d90f6>] vmxnet3_poll_rx_only+0x36/0xa0 [vmxnet3]
[<ffffffffa263af1f>] net_rx_action+0x26f/0x390
[<ffffffffa20a0ff5>] __do_softirq+0xf5/0x280
[<ffffffffa20a11b8>] run_ksoftirqd+0x38/0x50
[<ffffffffa20ca6b4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffa20ca570>] ? lg_double_unlock+0x40/0x40
[<ffffffffa20c1d21>] kthread+0xd1/0xe0
[<ffffffffa20c1c50>] ? insert_kthread_work+0x40/0x40
[<ffffffffa2775c37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffa20c1c50>] ? insert_kthread_work+0x40/0x40
Code: 47 fe ff ff 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 66 90 b9 01 00 00 00 8b 17 85 d2 74 0d 83 fa 03 74 08 f3 90 <8b> 17 85 d2 75 f3 89 d0 f0 0f b1 0f 39 c2 75 e3 5d 66 90 c3 0f 
Complete report #56811
Frame # Function Binary Source or offset Line
1
nf_nat_setup_info
nf_nat 0xffffffffc089c3d2
Build id: 3.10.0-1062.4.3.el7.x86_64
0x2e2
2
__nf_nat_alloc_null_binding
nf_nat 0xffffffffc089c467
Build id: 3.10.0-1062.4.3.el7.x86_64
0x57
3
nf_nat_alloc_null_binding
nf_nat 0xffffffffc089c4b1
Build id: 3.10.0-1062.4.3.el7.x86_64
0x21
4
nf_nat_ipv4_fn
nf_nat_ipv4 0xffffffffc077278e
Build id: 3.10.0-1062.4.3.el7.x86_64
0x1be
5
? iptable_nat_ipv4_fn
iptable_nat 0xffffffffc08a8020
Build id: 3.10.0-1062.4.3.el7.x86_64
0x20
6
iptable_nat_ipv4_fn
iptable_nat 0xffffffffc08a8018
Build id: 3.10.0-1062.4.3.el7.x86_64
0x18
7
nf_iterate
vmlinux 0xffffffffaa08f418
Build id: 3.10.0-1062.4.3.el7.x86_64
0x98
8
nf_hook_slow
vmlinux 0xffffffffaa08f508
Build id: 3.10.0-1062.4.3.el7.x86_64
0xa8
9
ip_local_deliver
vmlinux 0xffffffffaa099bc6
Build id: 3.10.0-1062.4.3.el7.x86_64
0xc6
10
? ip_rcv_finish
vmlinux 0xffffffffaa0997b0
Build id: 3.10.0-1062.4.3.el7.x86_64
0x370
11
ip_rcv_finish
vmlinux 0xffffffffaa0994d0
Build id: 3.10.0-1062.4.3.el7.x86_64
0x90
12
ip_rcv
vmlinux 0xffffffffaa099ea0
Build id: 3.10.0-1062.4.3.el7.x86_64
0x2c0
13
? inet_del_offload
vmlinux 0xffffffffaa099440
Build id: 3.10.0-1062.4.3.el7.x86_64
0x40
14
__netif_receive_skb_core
vmlinux 0xffffffffaa04ebd9
Build id: 3.10.0-1062.4.3.el7.x86_64
0x729
15
? __getnstimeofday64
vmlinux 0xffffffffa9b06e6f
Build id: 3.10.0-1062.4.3.el7.x86_64
0x3f
16
__netif_receive_skb
vmlinux 0xffffffffaa04eed8
Build id: 3.10.0-1062.4.3.el7.x86_64
0x18
17
netif_receive_skb_internal
vmlinux 0xffffffffaa04ef60
Build id: 3.10.0-1062.4.3.el7.x86_64
0x40
18
napi_gro_receive
vmlinux 0xffffffffaa04fbe8
Build id: 3.10.0-1062.4.3.el7.x86_64
0xd8
19
vmxnet3_rq_rx_complete
vmxnet3 0xffffffffc03826b4
Build id: 3.10.0-1062.4.3.el7.x86_64
0x924
20
vmxnet3_poll_rx_only
vmxnet3 0xffffffffc03830e6
Build id: 3.10.0-1062.4.3.el7.x86_64
0x36
21
net_rx_action
vmlinux 0xffffffffaa04f57f
Build id: 3.10.0-1062.4.3.el7.x86_64
0x26f
22
__do_softirq
vmlinux 0xffffffffa9aa5305
Build id: 3.10.0-1062.4.3.el7.x86_64
0xf5
23
run_ksoftirqd
vmlinux 0xffffffffa9aa54c8
Build id: 3.10.0-1062.4.3.el7.x86_64
0x38
24
smpboot_thread_fn
vmlinux 0xffffffffa9acec24
Build id: 3.10.0-1062.4.3.el7.x86_64
0x144
25
? lg_double_unlock
vmlinux 0xffffffffa9aceae0
Build id: 3.10.0-1062.4.3.el7.x86_64
0x40
26
kthread
vmlinux 0xffffffffa9ac61f1
Build id: 3.10.0-1062.4.3.el7.x86_64
0xd1
27
? insert_kthread_work
vmlinux 0xffffffffa9ac6120
Build id: 3.10.0-1062.4.3.el7.x86_64
0x40
28
ret_from_fork_nospec_begin
vmlinux 0xffffffffaa18cd37
Build id: 3.10.0-1062.4.3.el7.x86_64
0x21
29
? insert_kthread_work
vmlinux 0xffffffffa9ac6120
Build id: 3.10.0-1062.4.3.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [ksoftirqd/2:19]
Modules linked in: xt_nat veth xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink xt_addrtype iptable_filter iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack br_netfilter bridge stp llc cmac arc4 md4 nls_utf8 cifs ccm dns_resolver overlay(T) vmw_vsock_vmci_transport vsock sb_edac coretemp iosf_mbi crc32_pclmul ppdev ghash_clmulni_intel vmw_balloon aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr sg joydev parport_pc parport i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic vmwgfx drm_kms_helper pata_acpi syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common drm crc32c_intel serio_raw mptspi ata_piix scsi_transport_spi mptscsih libata
mptbase vmxnet3 drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 2 PID: 19 Comm: ksoftirqd/2 Kdump: loaded Tainted: G             L ------------ T 3.10.0-1062.4.3.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/30/2014
task: ffff927cfb3f8000 ti: ffff927cfb3f4000 task.ti: ffff927cfb3f4000
RIP: 0010:[<ffffffffc089c3d2>]  [<ffffffffc089c3d2>] nf_nat_setup_info+0x2e2/0x320 [nf_nat]
RSP: 0018:ffff927cfb3f7930  EFLAGS: 00010246
RAX: ffff927c13c56758 RBX: ffff927c13c566e8 RCX: 0000000000000000
RDX: ffff927eb1a16998 RSI: 0000000082bd32e5 RDI: ffffffffc089f478
RBP: ffff927cfb3f79d0 R08: ffff927c13c566e8 R09: ffff927cfb3f7930
R10: ffff927cfb3f78e0 R11: ffffffffc051b5f8 R12: ffffffffaa7129c0
R13: ffff927c13c566e8 R14: ffffffffc089c01c R15: ffff927cfb3f7920
FS:  0000000000000000(0000) GS:ffff927ebfd00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f49a507c000 CR3: 00000003213e2000 CR4: 00000000001607e0
Call Trace:
[<ffffffffc089c467>] __nf_nat_alloc_null_binding+0x57/0x80 [nf_nat]
[<ffffffffc089c4b1>] nf_nat_alloc_null_binding+0x21/0x30 [nf_nat]
[<ffffffffc077278e>] nf_nat_ipv4_fn+0x1be/0x230 [nf_nat_ipv4]
[<ffffffffc08a8020>] ? iptable_nat_ipv4_fn+0x20/0x20 [iptable_nat]
[<ffffffffc08a8018>] iptable_nat_ipv4_fn+0x18/0x20 [iptable_nat]
[<ffffffffaa08f418>] nf_iterate+0x98/0xe0
[<ffffffffaa08f508>] nf_hook_slow+0xa8/0x110
[<ffffffffaa099bc6>] ip_local_deliver+0xc6/0xe0
[<ffffffffaa0997b0>] ? ip_rcv_finish+0x370/0x370
[<ffffffffaa0994d0>] ip_rcv_finish+0x90/0x370
[<ffffffffaa099ea0>] ip_rcv+0x2c0/0x420
[<ffffffffaa099440>] ? inet_del_offload+0x40/0x40
[<ffffffffaa04ebd9>] __netif_receive_skb_core+0x729/0xa10
[<ffffffffa9b06e6f>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffffaa04eed8>] __netif_receive_skb+0x18/0x60
[<ffffffffaa04ef60>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffaa04fbe8>] napi_gro_receive+0xd8/0x100
[<ffffffffc03826b4>] vmxnet3_rq_rx_complete+0x924/0xef0 [vmxnet3]
[<ffffffffc03830e6>] vmxnet3_poll_rx_only+0x36/0xa0 [vmxnet3]
[<ffffffffaa04f57f>] net_rx_action+0x26f/0x390
[<ffffffffa9aa5305>] __do_softirq+0xf5/0x280
[<ffffffffa9aa54c8>] run_ksoftirqd+0x38/0x50
[<ffffffffa9acec24>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffa9aceae0>] ? lg_double_unlock+0x40/0x40
[<ffffffffa9ac61f1>] kthread+0xd1/0xe0
[<ffffffffa9ac6120>] ? insert_kthread_work+0x40/0x40
[<ffffffffaa18cd37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffa9ac6120>] ? insert_kthread_work+0x40/0x40
Code: d2 74 56 0f b7 42 12 66 85 c0 74 4d 48 01 d0 48 89 58 10 49 8b 96 28 0b 00 00 4a 8d 14 e2 48 8b 0a 48 89 50 08 48 89 08 48 85 c9 <48> 89 02 74 04 48 89 41 08 48 c7 c7 78 f4 89 c0 e8 c9 5e 8e e9 
Complete report #21492
Frame # Function Binary Source or offset Line
1
nf_nat_setup_info
nf_nat 0xffffffffc08213d2
Build id: 3.10.0-1062.9.1.el7.x86_64
0x2e2
2
__nf_nat_alloc_null_binding
nf_nat 0xffffffffc0821467
Build id: 3.10.0-1062.9.1.el7.x86_64
0x57
3
nf_nat_alloc_null_binding
nf_nat 0xffffffffc08214b1
Build id: 3.10.0-1062.9.1.el7.x86_64
0x21
4
nf_nat_ipv4_fn
nf_nat_ipv4 0xffffffffc080378e
Build id: 3.10.0-1062.9.1.el7.x86_64
0x1be
5
? iptable_nat_ipv4_fn
iptable_nat 0xffffffffc0828020
Build id: 3.10.0-1062.9.1.el7.x86_64
0x20
6
iptable_nat_ipv4_fn
iptable_nat 0xffffffffc0828018
Build id: 3.10.0-1062.9.1.el7.x86_64
0x18
7
nf_iterate
vmlinux 0xffffffffae690418
Build id: 3.10.0-1062.9.1.el7.x86_64
0x98
8
nf_hook_slow
vmlinux 0xffffffffae690508
Build id: 3.10.0-1062.9.1.el7.x86_64
0xa8
9
ip_local_deliver
vmlinux 0xffffffffae69abc6
Build id: 3.10.0-1062.9.1.el7.x86_64
0xc6
10
? ip_rcv_finish
vmlinux 0xffffffffae69a7b0
Build id: 3.10.0-1062.9.1.el7.x86_64
0x370
11
ip_rcv_finish
vmlinux 0xffffffffae69a4d0
Build id: 3.10.0-1062.9.1.el7.x86_64
0x90
12
ip_rcv
vmlinux 0xffffffffae69aea0
Build id: 3.10.0-1062.9.1.el7.x86_64
0x2c0
13
? inet_del_offload
vmlinux 0xffffffffae69a440
Build id: 3.10.0-1062.9.1.el7.x86_64
0x40
14
__netif_receive_skb_core
vmlinux 0xffffffffae64fbd9
Build id: 3.10.0-1062.9.1.el7.x86_64
0x729
15
? tcp4_gro_receive
vmlinux 0xffffffffae6c7600
Build id: 3.10.0-1062.9.1.el7.x86_64
0x110
16
? __getnstimeofday64
vmlinux 0xffffffffae1076bf
Build id: 3.10.0-1062.9.1.el7.x86_64
0x3f
17
__netif_receive_skb
vmlinux 0xffffffffae64fed8
Build id: 3.10.0-1062.9.1.el7.x86_64
0x18
18
netif_receive_skb_internal
vmlinux 0xffffffffae64ff60
Build id: 3.10.0-1062.9.1.el7.x86_64
0x40
19
napi_gro_receive
vmlinux 0xffffffffae650be8
Build id: 3.10.0-1062.9.1.el7.x86_64
0xd8
20
vmxnet3_rq_rx_complete
vmxnet3 0xffffffffc041e6b4
Build id: 3.10.0-1062.9.1.el7.x86_64
0x924
21
vmxnet3_poll_rx_only
vmxnet3 0xffffffffc041f0e6
Build id: 3.10.0-1062.9.1.el7.x86_64
0x36
22
net_rx_action
vmlinux 0xffffffffae65057f
Build id: 3.10.0-1062.9.1.el7.x86_64
0x26f
23
__do_softirq
vmlinux 0xffffffffae0a5305
Build id: 3.10.0-1062.9.1.el7.x86_64
0xf5
24
run_ksoftirqd
vmlinux 0xffffffffae0a54c8
Build id: 3.10.0-1062.9.1.el7.x86_64
0x38
25
smpboot_thread_fn
vmlinux 0xffffffffae0cec24
Build id: 3.10.0-1062.9.1.el7.x86_64
0x144
26
? lg_double_unlock
vmlinux 0xffffffffae0ceae0
Build id: 3.10.0-1062.9.1.el7.x86_64
0x40
27
kthread
vmlinux 0xffffffffae0c61f1
Build id: 3.10.0-1062.9.1.el7.x86_64
0xd1
28
? insert_kthread_work
vmlinux 0xffffffffae0c6120
Build id: 3.10.0-1062.9.1.el7.x86_64
0x40
29
ret_from_fork_nospec_begin
vmlinux 0xffffffffae78dd37
Build id: 3.10.0-1062.9.1.el7.x86_64
0x21
30
? insert_kthread_work
vmlinux 0xffffffffae0c6120
Build id: 3.10.0-1062.9.1.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 24s! [ksoftirqd/0:6]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter tcp_diag inet_diag vmw_vsock_vmci_transport vsock sunrpc sb_edac coretemp iosf_mbi crc32_pclmul ppdev ghash_clmulni_intel vmw_balloon aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr sg joydev pcc_cpufreq parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic vmwgfx sr_mod cdrom drm_kms_helper ata_generic syscopyarea pata_acpi sysfillrect sysimgblt fb_sys_fops ttm drm mptspi ata_piix scsi_transport_spi mptscsih crct10dif_pclmul libata crct10dif_common
mptbase crc32c_intel serio_raw vmxnet3 drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 6 Comm: ksoftirqd/0 Kdump: loaded Tainted: G             L ------------   3.10.0-1062.9.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013
task: ffff9c95674e3150 ti: ffff9c95674f8000 task.ti: ffff9c95674f8000
RIP: 0010:[<ffffffffc08213d2>]  [<ffffffffc08213d2>] nf_nat_setup_info+0x2e2/0x320 [nf_nat]
RSP: 0018:ffff9c95674fb930  EFLAGS: 00010246
RAX: ffff9c9535725958 RBX: ffff9c95357258e8 RCX: 0000000000000000
RDX: ffff9c960bff7288 RSI: 000000009e6f06e1 RDI: ffffffffc0824478
RBP: ffff9c95674fb9d0 R08: ffff9c95357258e8 R09: ffff9c95674fb930
R10: ffff9c95674fb8e0 R11: ffffffffc04cb5f8 R12: ffffffffaed13a40
R13: ffff9c95357258e8 R14: ffffffffc082101c R15: ffff9c95674fb920
FS:  0000000000000000(0000) GS:ffff9c9568800000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000000b673b000 CR3: 00000000348c6000 CR4: 00000000000407f0
Call Trace:
[<ffffffffc0821467>] __nf_nat_alloc_null_binding+0x57/0x80 [nf_nat]
[<ffffffffc08214b1>] nf_nat_alloc_null_binding+0x21/0x30 [nf_nat]
[<ffffffffc080378e>] nf_nat_ipv4_fn+0x1be/0x230 [nf_nat_ipv4]
[<ffffffffc0828020>] ? iptable_nat_ipv4_fn+0x20/0x20 [iptable_nat]
[<ffffffffc0828018>] iptable_nat_ipv4_fn+0x18/0x20 [iptable_nat]
[<ffffffffae690418>] nf_iterate+0x98/0xe0
[<ffffffffae690508>] nf_hook_slow+0xa8/0x110
[<ffffffffae69abc6>] ip_local_deliver+0xc6/0xe0
[<ffffffffae69a7b0>] ? ip_rcv_finish+0x370/0x370
[<ffffffffae69a4d0>] ip_rcv_finish+0x90/0x370
[<ffffffffae69aea0>] ip_rcv+0x2c0/0x420
[<ffffffffae69a440>] ? inet_del_offload+0x40/0x40
[<ffffffffae64fbd9>] __netif_receive_skb_core+0x729/0xa10
[<ffffffffae6c7600>] ? tcp4_gro_receive+0x110/0x1a0
[<ffffffffae1076bf>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffffae64fed8>] __netif_receive_skb+0x18/0x60
[<ffffffffae64ff60>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffae650be8>] napi_gro_receive+0xd8/0x100
[<ffffffffc041e6b4>] vmxnet3_rq_rx_complete+0x924/0xef0 [vmxnet3]
[<ffffffffc041f0e6>] vmxnet3_poll_rx_only+0x36/0xa0 [vmxnet3]
[<ffffffffae65057f>] net_rx_action+0x26f/0x390
[<ffffffffae0a5305>] __do_softirq+0xf5/0x280
[<ffffffffae0a54c8>] run_ksoftirqd+0x38/0x50
[<ffffffffae0cec24>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffae0ceae0>] ? lg_double_unlock+0x40/0x40
[<ffffffffae0c61f1>] kthread+0xd1/0xe0
[<ffffffffae0c6120>] ? insert_kthread_work+0x40/0x40
[<ffffffffae78dd37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffae0c6120>] ? insert_kthread_work+0x40/0x40
Code: d2 74 56 0f b7 42 12 66 85 c0 74 4d 48 01 d0 48 89 58 10 49 8b 96 28 0b 00 00 4a 8d 14 e2 48 8b 0a 48 89 50 08 48 89 08 48 85 c9 <48> 89 02 74 04 48 89 41 08 48 c7 c7 78 44 82 c0 e8 c9 1d f6 ed