Complete report #1153426
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
eth_type_trans
|
vmlinux |
0xffffffffa42aceb5
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x55
|
|
2
|
vmxnet3_rq_rx_complete
|
vmxnet3 |
0xffffffffc08c754b
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x74b
|
|
3
|
vmxnet3_poll_rx_only
|
vmxnet3 |
0xffffffffc08c8176
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x36
|
|
4
|
net_rx_action
|
vmlinux |
0xffffffffa4280037
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x277
|
|
5
|
__do_softirq
|
vmlinux |
0xffffffffa3ca9595
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0xf5
|
|
6
|
run_ksoftirqd
|
vmlinux |
0xffffffffa3ca9768
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x38
|
|
7
|
smpboot_thread_fn
|
vmlinux |
0xffffffffa3cd43fc
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x14c
|
|
8
|
|
vmlinux |
0xffffffffa3cd42b0
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x40
|
|
9
|
kthread
|
vmlinux |
0xffffffffa3ccb621
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0xd1
|
|
10
|
|
vmlinux |
0xffffffffa3ccb550
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x40
|
|
11
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffa43c51dd
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x7
|
|
12
|
|
vmlinux |
0xffffffffa3ccb550
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#7 stuck for 23s! [ksoftirqd/7:45]
Modules linked in: tcp_diag udp_diag inet_diag binfmt_misc xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun xt_comment bridge stp llc ebtable_filter ebtables ip6table_filter devlink ip6_tables iptable_filter ipmi_devintf ipmi_msghandler vmw_vsock_vmci_transport vsock sunrpc vfat fat sb_edac intel_powerclamp iosf_mbi crc32_pclmul ppdev vmw_balloon ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr vmxnet3 i2c_piix4 sg vmw_vmci parport_pc parport tpm_crb ip_tables xfs libcrc32c ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops sd_mod ttm crc_t10dif crct10dif_generic ata_piix
drm nfit libata mptsas libnvdimm crct10dif_pclmul crct10dif_common scsi_transport_sas crc32c_intel mptscsih serio_raw mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse [last unloaded: rpm_deathwatch]
CPU: 7 PID: 45 Comm: ksoftirqd/7 Kdump: loaded Tainted: G OE ------------ 3.10.0-1160.88.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware7,1/440BX Desktop Reference Platform, BIOS VMW71.00V.21100432.B64.2301110304 01/11/2023
task: ffff8c2a8206a100 ti: ffff8c2a82074000 task.ti: ffff8c2a82074000
RIP: 0010:[<ffffffffa42aceb5>] [<ffffffffa42aceb5>] eth_type_trans+0x55/0xe0
RSP: 0000:ffff8c2a82077cd0 EFLAGS: 00010286
RAX: ffff8c5983295a80 RBX: 0000000000000780 RCX: ffff8c5983295ace
RDX: ffff8c5983295ac0 RSI: ffff8c414c5a4000 RDI: ffff8c59bf585800
RBP: ffff8c2a82077cd0 R08: 00000000000000fe R09: 0000000000000000
R10: ffff8c59c1f1ac10 R11: ffff8c414c5a48c0 R12: ffff8c414c5a5ae0
R13: ffff8c59bf584e00 R14: ffff8c59bf584e00 R15: 00000000b522027c
FS: 0000000000000000(0000) GS:ffff8c419ddc0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000055c123857f50 CR3: 0000002634010000 CR4: 00000000000607e0
Call Trace:
[<ffffffffc08c754b>] vmxnet3_rq_rx_complete+0x74b/0xf00 [vmxnet3]
[<ffffffffc08c8176>] vmxnet3_poll_rx_only+0x36/0xa0 [vmxnet3]
[<ffffffffa4280037>] net_rx_action+0x277/0x3a0
[<ffffffffa3ca9595>] __do_softirq+0xf5/0x290
[<ffffffffa3ca9768>] run_ksoftirqd+0x38/0x60
[<ffffffffa3cd43fc>] smpboot_thread_fn+0x14c/0x1b0
[<ffffffffa3cd42b0>] ? lg_double_unlock+0x40/0x40
[<ffffffffa3ccb621>] kthread+0xd1/0xe0
[<ffffffffa3ccb550>] ? insert_kthread_work+0x40/0x40
[<ffffffffa43c51dd>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffffa3ccb550>] ? insert_kthread_work+0x40/0x40
Code: 89 97 c4 00 00 00 76 1d 41 83 e8 0e 44 3b 47 6c 44 89 47 68 0f 82 8f 00 00 00 48 83 c1 0e 48 89 8f e8 00 00 00 0f b7 d2 48 01 c2 <f6> 02 01 75 57 48 8b 8e 98 02 00 00 48 8b 02 48 33 01 48 c1 e0
Complete report #759218
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
eth_type_trans
|
vmlinux |
0xffffffff874862a5
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x55
|
|
2
|
vmxnet3_rq_rx_complete
|
vmxnet3 |
0xffffffffc032c4d3
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x743
|
|
3
|
vmxnet3_poll_rx_only
|
vmxnet3 |
0xffffffffc032d0e6
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x36
|
|
4
|
net_rx_action
|
vmlinux |
0xffffffff8745a1cf
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x26f
|
|
5
|
__do_softirq
|
vmlinux |
0xffffffff86ea4bf5
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xf5
|
|
6
|
run_ksoftirqd
|
vmlinux |
0xffffffff86ea4db8
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x38
|
|
7
|
smpboot_thread_fn
|
vmlinux |
0xffffffff86ece8b4
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x144
|
|
8
|
|
vmlinux |
0xffffffff86ece770
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x40
|
|
9
|
kthread
|
vmlinux |
0xffffffff86ec5e61
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xd1
|
|
10
|
|
vmlinux |
0xffffffff86ec5d90
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x40
|
|
11
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff87599df7
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x21
|
|
12
|
|
vmlinux |
0xffffffff86ec5d90
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#9 stuck for 21s! [ksoftirqd/9:54]
Modules linked in: falcon_lsm_serviceable(PE) falcon_nf_netcontain(PE) falcon_kal(E) falcon_lsm_pinned_15309(E) falcon_lsm_pinned_15110(E) falcon_lsm_pinned_15003(E) falcon_lsm_pinned_14812(E) falcon_lsm_pinned_14713(E) nf_conntrack_netlink xt_addrtype xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat xt_conntrack ipt_REJECT nf_reject_ipv4 tun ebtable_filter ebtables ip6table_filter devlink ip6_tables iptable_filter overlay(T) falcon_lsm_pinned_14604(E) vmw_vsock_vmci_transport vsock sunrpc sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev sg vmw_vmci vmw_balloon pcspkr joydev i2c_piix4 parport_pc parport nf_conntrack_ipv4 ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs nf_conntrack ip_tables
xfs sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops sd_mod ttm crc_t10dif crct10dif_generic ata_piix drm crct10dif_pclmul crct10dif_common nfit libata crc32c_intel serio_raw libnvdimm vmxnet3 vmw_pvscsi drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod fuse nf_defrag_ipv4 br_netfilter bridge stp llc libcrc32c [last unloaded: falcon_kal]
CPU: 9 PID: 54 Comm: ksoftirqd/9 Kdump: loaded Tainted: P EL ------------ T 3.10.0-1160.59.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff9d4835d81080 ti: ffff9d4835d88000 task.ti: ffff9d4835d88000
RIP: 0010:[<ffffffff874862a5>] [<ffffffff874862a5>] eth_type_trans+0x55/0xe0
RSP: 0000:ffff9d4835d8bcd0 EFLAGS: 00010286
RAX: ffff9d4763a04b80 RBX: 0000000000000780 RCX: ffff9d4763a04bce
RDX: ffff9d4763a04bc0 RSI: ffff9d4ae8458000 RDI: ffff9d477b462700
RBP: ffff9d4835d8bcd0 R08: 0000000000000094 R09: 0000000000000000
R10: ffff9d477a4926d0 R11: ffff9d4ae84588c0 R12: ffff9d4835d8bd40
R13: 0000000000007692 R14: ffff9d4835de4800 R15: ffff9d4835de4c00
FS: 0000000000000000(0000) GS:ffff9d4aef440000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f8521077000 CR3: 0000000428fac000 CR4: 00000000003607e0
Call Trace:
[<ffffffffc032c4d3>] vmxnet3_rq_rx_complete+0x743/0xef0 [vmxnet3]
[<ffffffffc032d0e6>] vmxnet3_poll_rx_only+0x36/0xa0 [vmxnet3]
[<ffffffff8745a1cf>] net_rx_action+0x26f/0x390
[<ffffffff86ea4bf5>] __do_softirq+0xf5/0x280
[<ffffffff86ea4db8>] run_ksoftirqd+0x38/0x50
[<ffffffff86ece8b4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff86ece770>] ? lg_double_unlock+0x40/0x40
[<ffffffff86ec5e61>] kthread+0xd1/0xe0
[<ffffffff86ec5d90>] ? insert_kthread_work+0x40/0x40
[<ffffffff87599df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff86ec5d90>] ? insert_kthread_work+0x40/0x40
Code: 89 97 c4 00 00 00 76 1d 41 83 e8 0e 44 3b 47 6c 44 89 47 68 0f 82 87 00 00 00 48 83 c1 0e 48 89 8f e8 00 00 00 0f b7 d2 48 01 c2 <f6> 02 01 75 4f 48 8b 8e 98 02 00 00 48 8b 02 48 33 01 48 c1 e0
Complete report #220729
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
eth_type_trans
|
vmlinux |
0xffffffff815ae735
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x55
|
|
2
|
vmxnet3_rq_rx_complete
|
vmxnet3 |
0xffffffffc0068bf3
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x473
|
|
3
|
vmxnet3_poll_rx_only
|
vmxnet3 |
0xffffffffc006950a
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x3a
|
|
4
|
net_rx_action
|
vmlinux |
0xffffffff8158799d
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x16d
|
|
5
|
__do_softirq
|
vmlinux |
0xffffffff81090b3f
Build id: 3.10.0-693.2.2.el7.x86_64
|
0xef
|
|
6
|
run_ksoftirqd
|
vmlinux |
0xffffffff81090d08
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x38
|
|
7
|
smpboot_thread_fn
|
vmlinux |
0xffffffff810b909f
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x12f
|
|
8
|
|
vmlinux |
0xffffffff810b8f70
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x40
|
|
9
|
kthread
|
vmlinux |
0xffffffff810b098f
Build id: 3.10.0-693.2.2.el7.x86_64
|
0xcf
|
|
10
|
|
vmlinux |
0xffffffff810b08c0
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x40
|
|
11
|
ret_from_fork
|
vmlinux |
0xffffffff816b4f58
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x58
|
|
12
|
|
vmlinux |
0xffffffff810b08c0
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 29s! [ksoftirqd/3:23]
Modules linked in: fuse xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock ext4 mbcache jbd2 ppdev pcspkr vmw_balloon sg joydev parport_pc parport shpchp vmw_vmci i2c_piix4 nfsd auth_rpcgss nfs_acl lockd grace sunrpc binfmt_misc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic crct10dif_common ata_generic
pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect crc32c_intel sysimgblt fb_sys_fops ttm mptspi scsi_transport_spi drm mptscsih serio_raw ata_piix libata mptbase vmxnet3 i2c_core floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 3 PID: 23 Comm: ksoftirqd/3 Tainted: G L ------------ 3.10.0-693.2.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff88017cae4f10 ti: ffff88017cb00000 task.ti: ffff88017cb00000
RIP: 0010:[<ffffffff815ae735>] [<ffffffff815ae735>] eth_type_trans+0x55/0xe0
RSP: 0018:ffff88017cb03cd8 EFLAGS: 00010286
RAX: ffff8801bea7f880 RBX: ffff8800501d5200 RCX: ffff8801bea7f8ce
RDX: ffff8801bea7f8c0 RSI: ffff880230b28000 RDI: ffff88021aa89900
RBP: ffff88017cb03cd8 R08: 000000000000002e R09: 0000000000000000
R10: ffff8800ba21e3b0 R11: ffff8800b9a80588 R12: ffff8800501d5200
R13: ffff8800501d5200 R14: 00000000b22a6a16 R15: 0000000000000000
FS: 0000000000000000(0000) GS:ffff8802356c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 000000007ce4bbba CR3: 00000000b9be8000 CR4: 00000000000007e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff88017cb03d50 ffffffffc0068bf3 ffff880230b290e0 ffff8800b9a80588
ffff8800ba21e3b0 00000002000005dc 0000000000000000 0000004000000000
ffff880230b29180 0000003b00000000 ffff880230b288c0 ffff880230b290e0
Call Trace:
[<ffffffffc0068bf3>] vmxnet3_rq_rx_complete+0x473/0xc20 [vmxnet3]
[<ffffffffc006950a>] vmxnet3_poll_rx_only+0x3a/0xb0 [vmxnet3]
[<ffffffff8158799d>] net_rx_action+0x16d/0x380
[<ffffffff81090b3f>] __do_softirq+0xef/0x280
[<ffffffff81090d08>] run_ksoftirqd+0x38/0x50
[<ffffffff810b909f>] smpboot_thread_fn+0x12f/0x180
[<ffffffff810b8f70>] ? lg_double_unlock+0x40/0x40
[<ffffffff810b098f>] kthread+0xcf/0xe0
[<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
[<ffffffff816b4f58>] ret_from_fork+0x58/0x90
[<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
Code: 89 97 c4 00 00 00 76 1d 41 83 e8 0e 44 3b 47 6c 44 89 47 68 0f 82 87 00 00 00 48 83 c1 0e 48 89 8f e8 00 00 00 0f b7 d2 48 01 c2 <f6> 02 01 75 4f 48 8b 8e 98 02 00 00 48 8b 02 48 33 01 48 c1 e0
Complete report #20230
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
eth_type_trans
|
vmlinux |
0xffffffff8aa7a2b5
Build id: 3.10.0-1062.1.1.el7.x86_64
|
0x55
|
|
2
|
vmxnet3_rq_rx_complete
|
vmxnet3 |
0xffffffffc02ce4d3
Build id: 3.10.0-1062.1.1.el7.x86_64
|
0x743
|
|
3
|
|
vmlinux |
0xffffffff8a4e0000
Build id: 3.10.0-1062.1.1.el7.x86_64
|
0x580
|
|
4
|
vmxnet3_poll_rx_only
|
vmxnet3 |
0xffffffffc02cf0e6
Build id: 3.10.0-1062.1.1.el7.x86_64
|
0x36
|
|
5
|
net_rx_action
|
vmlinux |
0xffffffff8aa4e57f
Build id: 3.10.0-1062.1.1.el7.x86_64
|
0x26f
|
|
6
|
__do_softirq
|
vmlinux |
0xffffffff8a4a41e5
Build id: 3.10.0-1062.1.1.el7.x86_64
|
0xf5
|
|
7
|
run_ksoftirqd
|
vmlinux |
0xffffffff8a4a43a8
Build id: 3.10.0-1062.1.1.el7.x86_64
|
0x38
|
|
8
|
smpboot_thread_fn
|
vmlinux |
0xffffffff8a4cdb04
Build id: 3.10.0-1062.1.1.el7.x86_64
|
0x144
|
|
9
|
|
vmlinux |
0xffffffff8a4cd9c0
Build id: 3.10.0-1062.1.1.el7.x86_64
|
0x40
|
|
10
|
kthread
|
vmlinux |
0xffffffff8a4c50d1
Build id: 3.10.0-1062.1.1.el7.x86_64
|
0xd1
|
|
11
|
|
vmlinux |
0xffffffff8a4c5000
Build id: 3.10.0-1062.1.1.el7.x86_64
|
0x40
|
|
12
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff8ab8cd37
Build id: 3.10.0-1062.1.1.el7.x86_64
|
0x21
|
|
13
|
|
vmlinux |
0xffffffff8a4c5000
Build id: 3.10.0-1062.1.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 40s! [ksoftirqd/1:14]
Modules linked in: nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc iosf_mbi crc32_pclmul ghash_clmulni_intel vmw_balloon ppdev cryptd sg pcspkr joydev parport_pc vmw_vmci parport i2c_piix4 ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix libata drm mptspi scsi_transport_spi crct10dif_pclmul crct10dif_common mptscsih crc32c_intel serio_raw mptbase vmxnet3
drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 14 Comm: ksoftirqd/1 Kdump: loaded Tainted: G L ------------ 3.10.0-1062.1.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff99033c5fa0e0 ti: ffff99033c644000 task.ti: ffff99033c644000
RIP: 0010:[<ffffffff8aa7a2b5>] [<ffffffff8aa7a2b5>] eth_type_trans+0x55/0xe0
RSP: 0018:ffff99033c647cd0 EFLAGS: 00010286
RAX: ffff9902c5c5bc80 RBX: 0000000000000780 RCX: ffff9902c5c5bcce
RDX: ffff9902c5c5bcc0 RSI: ffff9902f5828000 RDI: ffff99032eb31100
RBP: ffff99033c647cd0 R08: 000000000000002e R09: 0000000000000000
R10: ffff99033a00c0e0 R11: ffff9902f58288c0 R12: ffff99032e83c300
R13: ffff99032e83c300 R14: ffff99033f801700 R15: 0000000070338947
FS: 0000000000000000(0000) GS:ffff99033fc80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fb7e7df0000 CR3: 000000007b266000 CR4: 00000000001607e0
Call Trace:
[<ffffffffc02ce4d3>] vmxnet3_rq_rx_complete+0x743/0xef0 [vmxnet3]
[<ffffffff8a4e0000>] ? select_task_rq_fair+0x580/0x760
[<ffffffffc02cf0e6>] vmxnet3_poll_rx_only+0x36/0xa0 [vmxnet3]
[<ffffffff8aa4e57f>] net_rx_action+0x26f/0x390
[<ffffffff8a4a41e5>] __do_softirq+0xf5/0x280
[<ffffffff8a4a43a8>] run_ksoftirqd+0x38/0x50
[<ffffffff8a4cdb04>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff8a4cd9c0>] ? lg_double_unlock+0x40/0x40
[<ffffffff8a4c50d1>] kthread+0xd1/0xe0
[<ffffffff8a4c5000>] ? insert_kthread_work+0x40/0x40
[<ffffffff8ab8cd37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff8a4c5000>] ? insert_kthread_work+0x40/0x40
Code: 89 97 c4 00 00 00 76 1d 41 83 e8 0e 44 3b 47 6c 44 89 47 68 0f 82 87 00 00 00 48 83 c1 0e 48 89 8f e8 00 00 00 0f b7 d2 48 01 c2 <f6> 02 01 75 4f 48 8b 8e 98 02 00 00 48 8b 02 48 33 01 48 c1 e0
Complete report #5009
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
eth_type_trans
|
vmlinux |
0xffffffff82c80515
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x55
|
|
2
|
vmxnet3_rq_rx_complete
|
vmxnet3 |
0xffffffffc039c4d3
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x743
|
|
3
|
vmxnet3_poll_rx_only
|
vmxnet3 |
0xffffffffc039d0e6
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x36
|
|
4
|
net_rx_action
|
vmlinux |
0xffffffff82c545bf
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x26f
|
|
5
|
__do_softirq
|
vmlinux |
0xffffffff826a5695
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xf5
|
|
6
|
run_ksoftirqd
|
vmlinux |
0xffffffff826a5858
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x38
|
|
7
|
smpboot_thread_fn
|
vmlinux |
0xffffffff826cf0e4
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x144
|
|
8
|
|
vmlinux |
0xffffffff826cefa0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x40
|
|
9
|
kthread
|
vmlinux |
0xffffffff826c6691
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xd1
|
|
10
|
|
vmlinux |
0xffffffff826c65c0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x40
|
|
11
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff82d92d37
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x21
|
|
12
|
|
vmlinux |
0xffffffff826c65c0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 34s! [ksoftirqd/4:29]
Modules linked in: rpcsec_gss_krb5 xt_nat veth xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink xt_addrtype iptable_nat nf_nat_ipv4 nf_nat overlay(T) vmw_vsock_vmci_transport vsock iptable_filter coretemp crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper vmw_vmci cryptd ppdev pcspkr sg vmw_balloon joydev parport_pc parport nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul drm crct10dif_common ata_piix crc32c_intel mptspi scsi_transport_spi mptscsih libata serio_raw mptbase vmxnet3 drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod nf_conntrack_ipv4
nf_defrag_ipv4 ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs nf_conntrack libcrc32c br_netfilter bridge stp llc
CPU: 4 PID: 29 Comm: ksoftirqd/4 Tainted: G L ------------ T 3.10.0-1127.19.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/30/2014
task: ffff9f6ef4179070 ti: ffff9f6ef4180000 task.ti: ffff9f6ef4180000
RIP: 0010:[<ffffffff82c80515>] [<ffffffff82c80515>] eth_type_trans+0x55/0xe0
RSP: 0018:ffff9f6ef4183cd0 EFLAGS: 00010286
RAX: ffff9f6dd8953c80 RBX: 0000000000000780 RCX: ffff9f6dd8953cce
RDX: ffff9f6dd8953cc0 RSI: ffff9f7596c08000 RDI: ffff9f74f43c5400
RBP: ffff9f6ef4183cd0 R08: 000000000000002e R09: 0000000000000000
R10: ffff9f757f4c45a8 R11: ffff9f7596c088c0 R12: ffff9f7596c090e0
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 34s! [grafana-server:64414]
R13: ffff9f73980d3e00 R14: ffff9f73980d3e00 R15: 00000000c9384c76
FS: 0000000000000000(0000) GS:ffff9f759d700000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f61d8c60bd0 CR3: 00000007f9f78000 CR4: 00000000000007e0
Modules linked in: rpcsec_gss_krb5 xt_nat veth xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink xt_addrtype iptable_nat nf_nat_ipv4 nf_nat overlay(T) vmw_vsock_vmci_transport vsock iptable_filter coretemp crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper vmw_vmci cryptd ppdev pcspkr sg vmw_balloon joydev parport_pc parport nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul drm crct10dif_common ata_piix crc32c_intel mptspi scsi_transport_spi mptscsih libata serio_raw mptbase vmxnet3 drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod nf_conntrack_ipv4
nf_defrag_ipv4 ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs nf_conntrack libcrc32c br_netfilter bridge stp llc
CPU: 2 PID: 64414 Comm: grafana-server Tainted: G L ------------ T 3.10.0-1127.19.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/30/2014
task: ffff9f738ab8d230 ti: ffff9f7353a34000 task.ti: ffff9f7353a34000
RIP: 0033:[<0000000000433a52>] [<0000000000433a52>] 0x433a52
RSP: 002b:000000c000315ce0 EFLAGS: 00010202
RAX: 00007f497e872f18 RBX: 000000c00108a800 RCX: 000000c00029a780
RDX: 0000000000000000 RSI: 0000000000000001 RDI: 000000c00007a848
RBP: 000000c000315cf0 R08: 0000000000000000 R09: 0000000000000000
R10: 0000000000000000 R11: 0000000000000202 R12: ffff9f759d6861e8
R13: 000000c000d02180 R14: 000000c0000461e0 R15: 0000000000000000
FS: 00007f497e9bcb20(0000) GS:ffff9f759d680000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c00108b010 CR3: 000000006355a000 CR4: 00000000000007e0
Call Trace:
[<ffffffffc039c4d3>] vmxnet3_rq_rx_complete+0x743/0xef0 [vmxnet3]
[<ffffffffc039d0e6>] vmxnet3_poll_rx_only+0x36/0xa0 [vmxnet3]
[<ffffffff82c545bf>] net_rx_action+0x26f/0x390
[<ffffffff826a5695>] __do_softirq+0xf5/0x280
[<ffffffff826a5858>] run_ksoftirqd+0x38/0x50
[<ffffffff826cf0e4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff826cefa0>] ? lg_double_unlock+0x40/0x40
[<ffffffff826c6691>] kthread+0xd1/0xe0
[<ffffffff826c65c0>] ? insert_kthread_work+0x40/0x40
[<ffffffff82d92d37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff826c65c0>] ? insert_kthread_work+0x40/0x40
Code: 89 97 c4 00 00 00 76 1d 41 83 e8 0e 44 3b 47 6c 44 89 47 68 0f 82 87 00 00 00 48 83 c1 0e 48 89 8f e8 00 00 00 0f b7 d2 48 01 c2 <f6> 02 01 75 4f 48 8b 8e 98 02 00 00 48 8b 02 48 33 01 48 c1 e0
Complete report #10434
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
__build_skb
|
vmlinux |
0xffffffff95241d2b
Build id: 3.10.0-1127.8.2.el7.x86_64
|
0xdb
|
|
2
|
__netdev_alloc_skb
|
vmlinux |
0xffffffff95241f69
Build id: 3.10.0-1127.8.2.el7.x86_64
|
0xc9
|
|
3
|
vmxnet3_rq_rx_complete
|
vmxnet3 |
0xffffffffc04582de
Build id: 3.10.0-1127.8.2.el7.x86_64
|
0x54e
|
|
4
|
vmxnet3_poll_rx_only
|
vmxnet3 |
0xffffffffc04590e6
Build id: 3.10.0-1127.8.2.el7.x86_64
|
0x36
|
|
5
|
net_rx_action
|
vmlinux |
0xffffffff952545bf
Build id: 3.10.0-1127.8.2.el7.x86_64
|
0x26f
|
|
6
|
__do_softirq
|
vmlinux |
0xffffffff94ca5695
Build id: 3.10.0-1127.8.2.el7.x86_64
|
0xf5
|
|
7
|
run_ksoftirqd
|
vmlinux |
0xffffffff94ca5858
Build id: 3.10.0-1127.8.2.el7.x86_64
|
0x38
|
|
8
|
smpboot_thread_fn
|
vmlinux |
0xffffffff94ccf0e4
Build id: 3.10.0-1127.8.2.el7.x86_64
|
0x144
|
|
9
|
|
vmlinux |
0xffffffff94ccefa0
Build id: 3.10.0-1127.8.2.el7.x86_64
|
0x40
|
|
10
|
kthread
|
vmlinux |
0xffffffff94cc6691
Build id: 3.10.0-1127.8.2.el7.x86_64
|
0xd1
|
|
11
|
|
vmlinux |
0xffffffff94cc65c0
Build id: 3.10.0-1127.8.2.el7.x86_64
|
0x40
|
|
12
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff95392d37
Build id: 3.10.0-1127.8.2.el7.x86_64
|
0x21
|
|
13
|
|
vmlinux |
0xffffffff94cc65c0
Build id: 3.10.0-1127.8.2.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#12 stuck for 21s! [ksoftirqd/12:69]
Modules linked in: veth dummy vport_vxlan vxlan ip6_udp_tunnel udp_tunnel openvswitch xt_nat xt_statistic ipt_REJECT nf_reject_ipv4 xt_NFLOG xt_physdev xt_state nfnetlink_log ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs ip_set_hash_ip xt_set ip_set nf_tables ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6_tables xt_comment xt_mark ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 xt_addrtype iptable_filter xt_conntrack nf_nat nf_conntrack br_netfilter bridge stp llc overlay(T) vmw_vsock_vmci_transport vsock sunrpc sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel ppdev aesni_intel vmw_balloon lrw gf128mul glue_helper ablk_helper cryptd pcspkr joydev sg vmw_vmci i2c_piix4 parport_pc parport ip_tables xfs libcrc32c
sr_mod cdrom ata_generic pata_acpi vmwgfx sd_mod crc_t10dif crct10dif_generic drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci crct10dif_pclmul crct10dif_common crc32c_intel ata_piix libahci drm libata nfit serio_raw vmxnet3 libnvdimm vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 12 PID: 69 Comm: ksoftirqd/12 Kdump: loaded Tainted: G L ------------ T 3.10.0-1127.8.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff9699d2e362a0 ti: ffff9699d2e78000 task.ti: ffff9699d2e78000
RIP: 0010:[<ffffffff95241d2b>] [<ffffffff95241d2b>] __build_skb+0xdb/0x1b0
RSP: 0000:ffff9699d2e7bc88 EFLAGS: 00010286
RAX: 00000000ffffffff RBX: ffffffff95966c40 RCX: 00000000ffffffff
RDX: ffff96b3be9fae00 RSI: 0000000000000640 RDI: ffff96b36fcf6fc0
RBP: ffff9699d2e7bc98 R08: 000000000001f120 R09: 0000000000000000
R10: ffff96b82953a298 R11: ffff96b83740c8c0 R12: ffffffff95253c19
R13: ffff9699d2e7bc58 R14: ffffffff959696a0 R15: ffff96b83740e478
FS: 0000000000000000(0000) GS:ffff96b83d900000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f52569b3a10 CR3: 0000001f13340000 CR4: 00000000001607e0
Call Trace:
[<ffffffff95241f69>] __netdev_alloc_skb+0xc9/0x110
[<ffffffffc04582de>] vmxnet3_rq_rx_complete+0x54e/0xef0 [vmxnet3]
[<ffffffffc04590e6>] vmxnet3_poll_rx_only+0x36/0xa0 [vmxnet3]
[<ffffffff952545bf>] net_rx_action+0x26f/0x390
[<ffffffff94ca5695>] __do_softirq+0xf5/0x280
[<ffffffff94ca5858>] run_ksoftirqd+0x38/0x50
[<ffffffff94ccf0e4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff94ccefa0>] ? lg_double_unlock+0x40/0x40
[<ffffffff94cc6691>] kthread+0xd1/0xe0
[<ffffffff94cc65c0>] ? insert_kthread_work+0x40/0x40
[<ffffffff95392d37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff94cc65c0>] ? insert_kthread_work+0x40/0x40
Code: 89 a2 e0 00 00 00 4c 89 a2 e8 00 00 00 c7 82 d8 00 00 00 00 00 00 00 89 b2 dc 00 00 00 66 89 82 c4 00 00 00 66 89 8a c0 00 00 00 <48> c7 07 00 00 00 00 48 c7 47 08 00 00 00 00 48 c7 47 10 00 00
Complete report #1178870
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
__slab_alloc
|
vmlinux |
0xffffffff9e577617
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x48
|
|
2
|
kmem_cache_alloc
|
vmlinux |
0xffffffff9e0250db
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x19b
|
|
3
|
__build_skb
|
vmlinux |
0xffffffff9e43dc5a
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x2a
|
|
4
|
__netdev_alloc_skb
|
vmlinux |
0xffffffff9e43df49
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xc9
|
|
5
|
vmxnet3_rq_rx_complete
|
vmxnet3 |
0xffffffffc043e2de
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x54e
|
|
6
|
vmxnet3_poll_rx_only
|
vmxnet3 |
0xffffffffc043f0e6
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x36
|
|
7
|
net_rx_action
|
vmlinux |
0xffffffff9e45057f
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x26f
|
|
8
|
__do_softirq
|
vmlinux |
0xffffffff9dea5305
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xf5
|
|
9
|
run_ksoftirqd
|
vmlinux |
0xffffffff9dea54c8
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x38
|
|
10
|
smpboot_thread_fn
|
vmlinux |
0xffffffff9decec24
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x144
|
|
11
|
|
vmlinux |
0xffffffff9deceae0
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x40
|
|
12
|
kthread
|
vmlinux |
0xffffffff9dec61f1
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xd1
|
|
13
|
|
vmlinux |
0xffffffff9dec6120
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x40
|
|
14
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff9e58dd37
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x21
|
|
15
|
|
vmlinux |
0xffffffff9dec6120
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 21s! [ksoftirqd/0:6]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi vmw_vsock_vmci_transport vsock zfs(POE) zunicode(POE) zavl(POE) icp(POE) ext4 zcommon(POE) mbcache jbd2 znvpair(POE) spl(OE) ppdev sb_edac vmw_balloon iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr joydev parport_pc parport i2c_piix4 vmw_vmci nfsd auth_rpcgss nfs_acl lockd grace sunrpc binfmt_misc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea
sysfillrect sd_mod sysimgblt fb_sys_fops ttm crc_t10dif crct10dif_generic drm crct10dif_pclmul crct10dif_common ata_piix crc32c_intel nfit serio_raw libata libnvdimm vmxnet3 vmw_pvscsi drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod sg
CPU: 0 PID: 6 Comm: ksoftirqd/0 Kdump: loaded Tainted: P OEL ------------ 3.10.0-1062.12.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff8ff73943d230 ti: ffff8ff7394e4000 task.ti: ffff8ff7394e4000
RIP: 0010:[<ffffffff9e577617>] [<ffffffff9e577617>] __slab_alloc+0x48/0x5c
RSP: 0018:ffff8ff7394e7c08 EFLAGS: 00000246
RAX: ffff8ff738a0b800 RBX: ffff8ff6b9e9d300 RCX: 0000000180100010
RDX: ffff8ff738a0b800 RSI: fffffda484e282c0 RDI: 0000000000000246
RBP: ffff8ff7394e7c38 R08: 0000000000000000 R09: 0000000180100010
R10: ffff8ff6b4a22b20 R11: ffff8ff6358c88c0 R12: ffffffff9e43dc5a
R13: ffffffff9e4376e7 R14: 0000000000000000 R15: ffff8ff6358c8000
FS: 0000000000000000(0000) GS:ffff8ff73fc00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fb04c005000 CR3: 0000000098a66000 CR4: 00000000001607f0
Call Trace:
[<ffffffff9e0250db>] kmem_cache_alloc+0x19b/0x1f0
[<ffffffff9e43dc5a>] __build_skb+0x2a/0x1b0
[<ffffffff9e43df49>] __netdev_alloc_skb+0xc9/0x110
[<ffffffffc043e2de>] vmxnet3_rq_rx_complete+0x54e/0xef0 [vmxnet3]
[<ffffffffc043f0e6>] vmxnet3_poll_rx_only+0x36/0xa0 [vmxnet3]
[<ffffffff9e45057f>] net_rx_action+0x26f/0x390
[<ffffffff9dea5305>] __do_softirq+0xf5/0x280
[<ffffffff9dea54c8>] run_ksoftirqd+0x38/0x50
[<ffffffff9decec24>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff9deceae0>] ? lg_double_unlock+0x40/0x40
[<ffffffff9dec61f1>] kthread+0xd1/0xe0
[<ffffffff9dec6120>] ? insert_kthread_work+0x40/0x40
[<ffffffff9e58dd37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff9dec6120>] ? insert_kthread_work+0x40/0x40
Code: 53 50 4c 89 45 d0 e8 89 8d aa ff 4c 8b 45 d0 48 89 c3 44 89 f2 4c 89 e7 4c 89 f9 44 89 ee e8 01 d3 aa ff 48 89 df 48 89 c2 57 9d <0f> 1f 44 00 00 48 89 d0 5a 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f
Complete report #1142369
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
__slab_alloc
|
vmlinux |
0xffffffffa957c8ac
Build id: 3.10.0-1127.8.2.el7.x86_64
|
0x48
|
|
2
|
kmem_cache_alloc
|
vmlinux |
0xffffffffa902882b
Build id: 3.10.0-1127.8.2.el7.x86_64
|
0x19b
|
|
3
|
__build_skb
|
vmlinux |
0xffffffffa9441c7a
Build id: 3.10.0-1127.8.2.el7.x86_64
|
0x2a
|
|
4
|
__netdev_alloc_skb
|
vmlinux |
0xffffffffa9441f69
Build id: 3.10.0-1127.8.2.el7.x86_64
|
0xc9
|
|
5
|
vmxnet3_rq_rx_complete
|
vmxnet3 |
0xffffffffc019a2de
Build id: 3.10.0-1127.8.2.el7.x86_64
|
0x54e
|
|
6
|
vmxnet3_poll_rx_only
|
vmxnet3 |
0xffffffffc019b0e6
Build id: 3.10.0-1127.8.2.el7.x86_64
|
0x36
|
|
7
|
net_rx_action
|
vmlinux |
0xffffffffa94545bf
Build id: 3.10.0-1127.8.2.el7.x86_64
|
0x26f
|
|
8
|
__do_softirq
|
vmlinux |
0xffffffffa8ea5695
Build id: 3.10.0-1127.8.2.el7.x86_64
|
0xf5
|
|
9
|
run_ksoftirqd
|
vmlinux |
0xffffffffa8ea5858
Build id: 3.10.0-1127.8.2.el7.x86_64
|
0x38
|
|
10
|
smpboot_thread_fn
|
vmlinux |
0xffffffffa8ecf0e4
Build id: 3.10.0-1127.8.2.el7.x86_64
|
0x144
|
|
11
|
|
vmlinux |
0xffffffffa8ecefa0
Build id: 3.10.0-1127.8.2.el7.x86_64
|
0x40
|
|
12
|
kthread
|
vmlinux |
0xffffffffa8ec6691
Build id: 3.10.0-1127.8.2.el7.x86_64
|
0xd1
|
|
13
|
|
vmlinux |
0xffffffffa8ec65c0
Build id: 3.10.0-1127.8.2.el7.x86_64
|
0x40
|
|
14
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffa9592d37
Build id: 3.10.0-1127.8.2.el7.x86_64
|
0x21
|
|
15
|
|
vmlinux |
0xffffffffa8ec65c0
Build id: 3.10.0-1127.8.2.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [ksoftirqd/4:29]
Modules linked in: xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 ipt_REJECT nf_reject_ipv4 iptable_filter xt_conntrack iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle ip_tables ip_set nfnetlink tun br_netfilter bridge stp llc devlink fuse binfmt_misc overlay(T) vmw_vsock_vmci_transport vsock sunrpc sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel ppdev lrw gf128mul glue_helper ablk_helper joydev cryptd vmw_balloon pcspkr sg vmw_vmci parport_pc parport i2c_piix4 xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops sd_mod ttm crc_t10dif crct10dif_generic drm ahci ata_piix libahci nfit crct10dif_pclmul crct10dif_common libnvdimm crc32c_intel libata serio_raw vmxnet3
vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod [last unloaded: ebtables]
CPU: 4 PID: 29 Comm: ksoftirqd/4 Kdump: loaded Tainted: G L ------------ T 3.10.0-1127.8.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff93d9bac45230 ti: ffff93d9bac68000 task.ti: ffff93d9bac68000
RIP: 0010:[<ffffffffa957c8ac>] [<ffffffffa957c8ac>] __slab_alloc+0x48/0x5c
RSP: 0000:ffff93d9bac6bc08 EFLAGS: 00000246
RAX: ffff93db173d4600 RBX: 0000000000000046 RCX: 0000000180100010
RDX: ffff93db173d4600 RSI: ffffd731cb5cf500 RDI: 0000000000000246
RBP: ffff93d9bac6bc38 R08: 0000000000000000 R09: 0000000180100010
R10: 00000000000036e0 R11: ffffffffffffffff R12: ffffffffa9441c7a
R13: ffffffff00000003 R14: 0000000000000000 R15: ffffffffffffffff
FS: 0000000000000000(0000) GS:ffff93db7fd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f5c57895000 CR3: 0000000023932000 CR4: 00000000003607e0
Call Trace:
[<ffffffffa902882b>] kmem_cache_alloc+0x19b/0x1f0
[<ffffffffa9441c7a>] __build_skb+0x2a/0x1b0
[<ffffffffa9441f69>] __netdev_alloc_skb+0xc9/0x110
[<ffffffffc019a2de>] vmxnet3_rq_rx_complete+0x54e/0xef0 [vmxnet3]
[<ffffffffc019b0e6>] vmxnet3_poll_rx_only+0x36/0xa0 [vmxnet3]
[<ffffffffa94545bf>] net_rx_action+0x26f/0x390
[<ffffffffa8ea5695>] __do_softirq+0xf5/0x280
[<ffffffffa8ea5858>] run_ksoftirqd+0x38/0x50
[<ffffffffa8ecf0e4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffa8ecefa0>] ? lg_double_unlock+0x40/0x40
[<ffffffffa8ec6691>] kthread+0xd1/0xe0
[<ffffffffa8ec65c0>] ? insert_kthread_work+0x40/0x40
[<ffffffffa9592d37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffa8ec65c0>] ? insert_kthread_work+0x40/0x40
Code: 53 50 4c 89 45 d0 e8 54 5e aa ff 4c 8b 45 d0 48 89 c3 44 89 f2 4c 89 e7 4c 89 f9 44 89 ee e8 fc a9 aa ff 48 89 df 48 89 c2 57 9d <0f> 1f 44 00 00 48 89 d0 5a 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f
Complete report #1001718
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
__build_skb
|
vmlinux |
0xffffffff97c27a8b
Build id: 3.10.0-957.el7.x86_64
|
0xdb
|
|
2
|
__netdev_alloc_skb
|
vmlinux |
0xffffffff97c27cc9
Build id: 3.10.0-957.el7.x86_64
|
0xc9
|
|
3
|
vmxnet3_rq_rx_complete
|
vmxnet3 |
0xffffffffc04905f6
Build id: 3.10.0-957.el7.x86_64
|
0x546
|
|
4
|
vmxnet3_poll_rx_only
|
vmxnet3 |
0xffffffffc04910f6
Build id: 3.10.0-957.el7.x86_64
|
0x36
|
|
5
|
net_rx_action
|
vmlinux |
0xffffffff97c39e9f
Build id: 3.10.0-957.el7.x86_64
|
0x26f
|
|
6
|
__do_softirq
|
vmlinux |
0xffffffff976a0f05
Build id: 3.10.0-957.el7.x86_64
|
0xf5
|
|
7
|
run_ksoftirqd
|
vmlinux |
0xffffffff976a10c8
Build id: 3.10.0-957.el7.x86_64
|
0x38
|
|
8
|
smpboot_thread_fn
|
vmlinux |
0xffffffff976ca5c4
Build id: 3.10.0-957.el7.x86_64
|
0x144
|
|
9
|
|
vmlinux |
0xffffffff976ca480
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
|
10
|
kthread
|
vmlinux |
0xffffffff976c1c31
Build id: 3.10.0-957.el7.x86_64
|
0xd1
|
|
11
|
|
vmlinux |
0xffffffff976c1b60
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
|
12
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff97d74c37
Build id: 3.10.0-957.el7.x86_64
|
0x21
|
|
13
|
|
vmlinux |
0xffffffff976c1b60
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [ksoftirqd/0:3]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink i2dtrack(OE) nbd(OE) dattobd(OE) sfs(OE) iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi vmw_vsock_vmci_transport vsock sunrpc iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev vmw_balloon pcspkr joydev sg parport_pc i2c_piix4 parport vmw_vmci binfmt_misc ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel serio_raw ata_generic pata_acpi floppy vmwgfx drm_kms_helper syscopyarea sysfillrect
sysimgblt fb_sys_fops ttm nfit vmxnet3 libnvdimm drm ata_piix mptspi scsi_transport_spi mptscsih mptbase drm_panel_orientation_quirks libata dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 3 Comm: ksoftirqd/0 Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff9b4ef94a2080 ti: ffff9b4ef94c8000 task.ti: ffff9b4ef94c8000
RIP: 0010:[<ffffffff97c27a8b>] [<ffffffff97c27a8b>] __build_skb+0xdb/0x1b0
RSP: 0018:ffff9b4ef94cbc88 EFLAGS: 00010282
RAX: 00000000ffffffff RBX: ffff9b4df56ce4a0 RCX: 00000000ffffffff
RDX: ffff9b4dc52b7000 RSI: 0000000000000640 RDI: ffff9b4e042ad940
RBP: ffff9b4ef94cbc98 R08: 000000000001f120 R09: 0000000000000000
R10: ffff9b4e76b68d38 R11: ffff9b4df56cc8c0 R12: ffffffff97c394f9
R13: ffff9b4ef94cbc58 R14: 0000000000000608 R15: ffff9b4df56cc000
FS: 0000000000000000(0000) GS:ffff9b4effc00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f89f76c4000 CR3: 000000008f79e000 CR4: 00000000001607f0
Call Trace:
[<ffffffff97c27cc9>] __netdev_alloc_skb+0xc9/0x110
[<ffffffffc04905f6>] vmxnet3_rq_rx_complete+0x546/0xef0 [vmxnet3]
[<ffffffffc04910f6>] vmxnet3_poll_rx_only+0x36/0xa0 [vmxnet3]
[<ffffffff97c39e9f>] net_rx_action+0x26f/0x390
[<ffffffff976a0f05>] __do_softirq+0xf5/0x280
[<ffffffff976a10c8>] run_ksoftirqd+0x38/0x50
[<ffffffff976ca5c4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff976ca480>] ? lg_double_unlock+0x40/0x40
[<ffffffff976c1c31>] kthread+0xd1/0xe0
[<ffffffff976c1b60>] ? insert_kthread_work+0x40/0x40
[<ffffffff97d74c37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff976c1b60>] ? insert_kthread_work+0x40/0x40
Code: 89 a2 e0 00 00 00 4c 89 a2 e8 00 00 00 c7 82 d8 00 00 00 00 00 00 00 89 b2 dc 00 00 00 66 89 82 c4 00 00 00 66 89 8a c0 00 00 00 <48> c7 07 00 00 00 00 48 c7 47 08 00 00 00 00 48 c7 47 10 00 00
Complete report #909970
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
__netdev_alloc_skb
|
vmlinux |
0xffffffffa6244fb4
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0xb4
|
|
2
|
vmxnet3_rq_rx_complete
|
vmxnet3 |
0xffffffffc021d2de
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x54e
|
|
3
|
vmxnet3_poll_rx_only
|
vmxnet3 |
0xffffffffc021e0e6
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x36
|
|
4
|
net_rx_action
|
vmlinux |
0xffffffffa62571cf
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x26f
|
|
5
|
__do_softirq
|
vmlinux |
0xffffffffa5ca4bf5
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0xf5
|
|
6
|
run_ksoftirqd
|
vmlinux |
0xffffffffa5ca4db8
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x38
|
|
7
|
smpboot_thread_fn
|
vmlinux |
0xffffffffa5cce8b4
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x144
|
|
8
|
|
vmlinux |
0xffffffffa5cce770
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x40
|
|
9
|
kthread
|
vmlinux |
0xffffffffa5cc5e61
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0xd1
|
|
10
|
|
vmlinux |
0xffffffffa5cc5d90
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x40
|
|
11
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffa6395ddd
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x7
|
|
12
|
|
vmlinux |
0xffffffffa5cc5d90
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 27s! [ksoftirqd/4:29]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter tun bridge stp llc nfsv3 nfs_acl nfs lockd grace fscache vmw_vsock_vmci_transport vsock dm_mirror dm_region_hash dm_log dm_mod iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev joydev sg vmw_balloon pcspkr i2c_piix4 vmw_vmci parport_pc parport auth_rpcgss sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm nfit sd_mod drm ahci crc_t10dif libnvdimm crct10dif_generic
libahci ata_piix libata crct10dif_pclmul crct10dif_common crc32c_intel serio_raw vmxnet3 vmw_pvscsi drm_panel_orientation_quirks fuse
CPU: 4 PID: 29 Comm: ksoftirqd/4 Kdump: loaded Tainted: G W L ------------ 3.10.0-1160.53.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff959038c55280 ti: ffff959038c78000 task.ti: ffff959038c78000
RIP: 0010:[<ffffffffa6244fb4>] [<ffffffffa6244fb4>] __netdev_alloc_skb+0xb4/0x110
RSP: 0018:ffff959038c7bca8 EFLAGS: 00000202
RAX: ffff958fc352f280 RBX: ffff95917ffda000 RCX: 0000000000007280
RDX: 0000000000008000 RSI: 0000000000007fff RDI: 0000000000000202
RBP: ffff959038c7bcd0 R08: ffffffffa668ed5f R09: ffffc9dbc40d4c00
R10: ffff95917fbd7900 R11: ffffffffffffffff R12: 00009591701a64a0
R13: 0000000200000003 R14: ffff95917ffdb000 R15: 0000004100000000
FS: 0000000000000000(0000) GS:ffff95917fd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00005646519b6000 CR3: 00000002b3f0a000 CR4: 00000000007607e0
PKRU: 00000000
Call Trace:
[<ffffffffc021d2de>] vmxnet3_rq_rx_complete+0x54e/0xef0 [vmxnet3]
[<ffffffffc021e0e6>] vmxnet3_poll_rx_only+0x36/0xa0 [vmxnet3]
[<ffffffffa62571cf>] net_rx_action+0x26f/0x390
[<ffffffffa5ca4bf5>] __do_softirq+0xf5/0x280
[<ffffffffa5ca4db8>] run_ksoftirqd+0x38/0x50
[<ffffffffa5cce8b4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffa5cce770>] ? lg_double_unlock+0x40/0x40
[<ffffffffa5cc5e61>] kthread+0xd1/0xe0
[<ffffffffa5cc5d90>] ? insert_kthread_work+0x40/0x40
[<ffffffffa6395ddd>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffffa5cc5d90>] ? insert_kthread_work+0x40/0x40
Code: 49 c7 c4 a0 9f 01 00 65 4c 03 25 48 c2 dc 59 4c 89 e7 44 89 c2 44 89 ee e8 6a 4a b8 ff 45 0f b6 64 24 10 49 89 c7 4c 89 f7 57 9d <0f> 1f 44 00 00 4d 85 ff 74 4a 44 89 ee 4c 89 ff e8 e7 fc ff ff
Complete report #840643
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
__netdev_alloc_skb
|
vmlinux |
0xffffffff85227cb4
Build id: 3.10.0-957.el7.x86_64
|
0xb4
|
|
2
|
vmxnet3_rq_rx_complete
|
vmxnet3 |
0xffffffffc016a5f6
Build id: 3.10.0-957.el7.x86_64
|
0x546
|
|
3
|
|
vmlinux |
0xffffffff84ce0000
Build id: 3.10.0-957.el7.x86_64
|
0x90
|
|
4
|
vmxnet3_poll_rx_only
|
vmxnet3 |
0xffffffffc016b0f6
Build id: 3.10.0-957.el7.x86_64
|
0x36
|
|
5
|
net_rx_action
|
vmlinux |
0xffffffff85239e9f
Build id: 3.10.0-957.el7.x86_64
|
0x26f
|
|
6
|
__do_softirq
|
vmlinux |
0xffffffff84ca0f05
Build id: 3.10.0-957.el7.x86_64
|
0xf5
|
|
7
|
run_ksoftirqd
|
vmlinux |
0xffffffff84ca10c8
Build id: 3.10.0-957.el7.x86_64
|
0x38
|
|
8
|
smpboot_thread_fn
|
vmlinux |
0xffffffff84cca5c4
Build id: 3.10.0-957.el7.x86_64
|
0x144
|
|
9
|
|
vmlinux |
0xffffffff84cca480
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
|
10
|
kthread
|
vmlinux |
0xffffffff84cc1c31
Build id: 3.10.0-957.el7.x86_64
|
0xd1
|
|
11
|
|
vmlinux |
0xffffffff84cc1b60
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
|
12
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff85374c1d
Build id: 3.10.0-957.el7.x86_64
|
0x7
|
|
13
|
|
vmlinux |
0xffffffff84cc1b60
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [ksoftirqd/6:39]
Modules linked in: tcp_lp fuse ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ebtable_nat ebtable_broute ip6table_security ip6table_raw iptable_security xt_set ipt_rpfilter iptable_raw ip_set_hash_net vxlan ip6_udp_tunnel udp_tunnel ip_set_hash_ip ip_set xt_multiport veth ip6t_MASQUERADE nf_nat_masquerade_ipv6 xt_statistic xt_nat xt_mark nf_tables(T) ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle xt_comment nf_conntrack_netlink nfnetlink xt_addrtype xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun ebtable_filter ebtables ip6table_filter devlink ip6_tables iptable_filter overlay(T) vmw_vsock_vmci_transport vsock sunrpc vfat fat iosf_mbi
crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr ppdev joydev vmw_balloon sg parport_pc parport tpm_crb vmw_vmci i2c_piix4 br_netfilter bridge stp llc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel serio_raw vmwgfx drm_kms_helper syscopyarea nfit sysfillrect sysimgblt fb_sys_fops ttm libnvdimm ahci libahci drm vmxnet3 ata_piix libata vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 6 PID: 39 Comm: ksoftirqd/6 Kdump: loaded Tainted: G L ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware7,1/440BX Desktop Reference Platform, BIOS VMW71.00V.16460286.B64.2006250725 06/25/2020
task: ffff9379b9cab0c0 ti: ffff9379b9cc0000 task.ti: ffff9379b9cc0000
RIP: 0010:[<ffffffff85227cb4>] [<ffffffff85227cb4>] __netdev_alloc_skb+0xb4/0x110
RSP: 0018:ffff9379b9cc3ca8 EFLAGS: 00000202
RAX: ffff937a3df6f880 RBX: ffff937c7ffda000 RCX: 0000000000007880
RDX: 0000000000008000 RSI: 0000000000007fff RDI: 0000000000000202
RBP: ffff9379b9cc3cd0 R08: ffffffff85681c99 R09: fffff6f887f7dc00
R10: ffff937c7fbd66e0 R11: ffff937c690108c0 R12: 0000000200000003
R13: 0000014100000000 R14: ffff937c7ffdb008 R15: ffff9379b9cab0c0
FS: 0000000000000000(0000) GS:ffff937c7fd80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f1cd01711c0 CR3: 000000000d1c2000 CR4: 00000000007607e0
PKRU: 00000000
Call Trace:
[<ffffffffc016a5f6>] vmxnet3_rq_rx_complete+0x546/0xef0 [vmxnet3]
[<ffffffff84ce0000>] ? update_curr+0x90/0x1e0
[<ffffffffc016b0f6>] vmxnet3_poll_rx_only+0x36/0xa0 [vmxnet3]
[<ffffffff85239e9f>] net_rx_action+0x26f/0x390
[<ffffffff84ca0f05>] __do_softirq+0xf5/0x280
[<ffffffff84ca10c8>] run_ksoftirqd+0x38/0x50
[<ffffffff84cca5c4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff84cca480>] ? lg_double_unlock+0x40/0x40
[<ffffffff84cc1c31>] kthread+0xd1/0xe0
[<ffffffff84cc1b60>] ? insert_kthread_work+0x40/0x40
[<ffffffff85374c1d>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffff84cc1b60>] ? insert_kthread_work+0x40/0x40
Code: 49 c7 c4 c0 9e 01 00 65 4c 03 25 08 95 de 7a 4c 89 e7 44 89 c2 44 89 ee e8 0a 94 b9 ff 45 0f b6 64 24 10 49 89 c7 4c 89 f7 57 9d <0f> 1f 44 00 00 4d 85 ff 74 4a 44 89 ee 4c 89 ff e8 e7 fc ff ff
Complete report #578619
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
__build_skb
|
vmlinux |
0xffffffff81577f3b
Build id: 3.10.0-693.2.2.el7.x86_64
|
0xdb
|
|
2
|
__netdev_alloc_skb
|
vmlinux |
0xffffffff81578179
Build id: 3.10.0-693.2.2.el7.x86_64
|
0xc9
|
|
3
|
vmxnet3_rq_rx_complete
|
vmxnet3 |
0xffffffffc006c919
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x199
|
|
4
|
vmxnet3_poll_rx_only
|
vmxnet3 |
0xffffffffc006d50a
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x3a
|
|
5
|
net_rx_action
|
vmlinux |
0xffffffff8158799d
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x16d
|
|
6
|
__do_softirq
|
vmlinux |
0xffffffff81090b3f
Build id: 3.10.0-693.2.2.el7.x86_64
|
0xef
|
|
7
|
run_ksoftirqd
|
vmlinux |
0xffffffff81090d08
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x38
|
|
8
|
smpboot_thread_fn
|
vmlinux |
0xffffffff810b909f
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x12f
|
|
9
|
|
vmlinux |
0xffffffff810b8f70
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x40
|
|
10
|
kthread
|
vmlinux |
0xffffffff810b098f
Build id: 3.10.0-693.2.2.el7.x86_64
|
0xcf
|
|
11
|
|
vmlinux |
0xffffffff810b08c0
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x40
|
|
12
|
ret_from_fork
|
vmlinux |
0xffffffff816b4f58
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x58
|
|
13
|
|
vmlinux |
0xffffffff810b08c0
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 23s! [ksoftirqd/0:3]
Modules linked in: gsch(OE) redirfs(OE) nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack iptable_filter nfsv3 rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache dsa_filter(POE) vmw_vsock_vmci_transport vsock sb_edac edac_core coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper vmw_balloon cryptd ppdev sg vmw_vmci joydev pcspkr parport_pc shpchp parport i2c_piix4 nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm crct10dif_pclmul crct10dif_common mptspi scsi_transport_spi mptscsih crc32c_intel serio_raw ata_piix libata mptbase vmxnet3 i2c_core dm_mirror dm_region_hash dm_log
dm_mod
CPU: 0 PID: 3 Comm: ksoftirqd/0 Tainted: P OEL ------------ 3.10.0-693.2.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/30/2014
task: ffff88017cd51fa0 ti: ffff88017cdcc000 task.ti: ffff88017cdcc000
RIP: 0010:[<ffffffff81577f3b>] [<ffffffff81577f3b>] __build_skb+0xdb/0x1b0
RSP: 0018:ffff88017cdcfc90 EFLAGS: 00010282
RAX: 00000000ffffffff RBX: ffff88023fc19c40 RCX: 00000000ffffffff
RDX: ffff88009ff51200 RSI: 0000000000000640 RDI: ffff88015edf8e40
RBP: ffff88017cdcfca0 R08: 0000000000019c40 R09: 0000000000000000
R10: ffff8800bb749040 R11: ffff8800b9b34060 R12: ffffffff81586f22
R13: ffff88017cdcfc68 R14: 0000000000000008 R15: ffffffff81b25798
FS: 0000000000000000(0000) GS:ffff88023fc00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f4915290930 CR3: 00000001f8bbd000 CR4: 00000000001407f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff880035d8c000 0000000000000000 ffff88017cdcfcd8 ffffffff81578179
ffff880035d8d1b8 ffff880035d8c8c0 ffff880035d8d0c0 ffff880216346800
ffff8800b9b6b750 ffff88017cdcfd50 ffffffffc006c919 ffff880035d8d0e0
Call Trace:
[<ffffffff81578179>] __netdev_alloc_skb+0xc9/0x110
[<ffffffffc006c919>] vmxnet3_rq_rx_complete+0x199/0xc20 [vmxnet3]
[<ffffffffc006d50a>] vmxnet3_poll_rx_only+0x3a/0xb0 [vmxnet3]
[<ffffffff8158799d>] net_rx_action+0x16d/0x380
[<ffffffff81090b3f>] __do_softirq+0xef/0x280
[<ffffffff81090d08>] run_ksoftirqd+0x38/0x50
[<ffffffff810b909f>] smpboot_thread_fn+0x12f/0x180
[<ffffffff810b8f70>] ? lg_double_unlock+0x40/0x40
[<ffffffff810b098f>] kthread+0xcf/0xe0
[<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
[<ffffffff816b4f58>] ret_from_fork+0x58/0x90
[<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
Code: 89 a2 e0 00 00 00 4c 89 a2 e8 00 00 00 c7 82 d8 00 00 00 00 00 00 00 89 b2 dc 00 00 00 66 89 82 c4 00 00 00 66 89 8a c0 00 00 00 <48> c7 07 00 00 00 00 48 c7 47 08 00 00 00 00 48 c7 47 10 00 00
Complete report #281453
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
kmem_cache_alloc
|
vmlinux |
0xffffffff86e1c384
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x74
|
|
2
|
__build_skb
|
vmlinux |
0xffffffff87228a5a
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x2a
|
|
3
|
__netdev_alloc_skb
|
vmlinux |
0xffffffff87228d49
Build id: 3.10.0-957.21.3.el7.x86_64
|
0xc9
|
|
4
|
vmxnet3_rq_rx_complete
|
vmxnet3 |
0xffffffffc01455f6
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x546
|
|
5
|
vmxnet3_poll_rx_only
|
vmxnet3 |
0xffffffffc01460f6
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x36
|
|
6
|
net_rx_action
|
vmlinux |
0xffffffff8723af1f
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x26f
|
|
7
|
__do_softirq
|
vmlinux |
0xffffffff86ca1075
Build id: 3.10.0-957.21.3.el7.x86_64
|
0xf5
|
|
8
|
run_ksoftirqd
|
vmlinux |
0xffffffff86ca1238
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x38
|
|
9
|
smpboot_thread_fn
|
vmlinux |
0xffffffff86cca734
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x144
|
|
10
|
|
vmlinux |
0xffffffff86cca5f0
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x40
|
|
11
|
kthread
|
vmlinux |
0xffffffff86cc1da1
Build id: 3.10.0-957.21.3.el7.x86_64
|
0xd1
|
|
12
|
|
vmlinux |
0xffffffff86cc1cd0
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x40
|
|
13
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff87375c37
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x21
|
|
14
|
|
vmlinux |
0xffffffff86cc1cd0
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 27s! [ksoftirqd/0:3]
Modules linked in: vmw_vsock_vmci_transport vsock binfmt_misc ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack iptable_filter ppdev vmw_balloon crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr joydev sg i2c_piix4 parport_pc parport vmw_vmci ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix mptspi crct10dif_pclmul scsi_transport_spi crct10dif_common libata crc32c_intel mptscsih serio_raw mptbase vmxnet3 drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 3 Comm: ksoftirqd/0 Kdump: loaded Tainted: G L ------------ 3.10.0-957.21.3.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff8e5dfa2aa080 ti: ffff8e5dfa334000 task.ti: ffff8e5dfa334000
RIP: 0010:[<ffffffff86e1c384>] [<ffffffff86e1c384>] kmem_cache_alloc+0x74/0x1f0
RSP: 0000:ffff8e5dfa337c48 EFLAGS: 00010282
RAX: 0000000000000000 RBX: 4006000000000000 RCX: 00000000e939600e
RDX: 00000000e939600d RSI: 0000000000000020 RDI: ffff8e5dffc03700
RBP: ffff8e5dfa337c78 R08: 000000000001f120 R09: ffffffff87228a5a
R10: ffff8e5f738b0d80 R11: ffff8e5cb64fc8c0 R12: 0000000000000000
R13: ffffffff872aa5f4 R14: ffff8e5dfa337c28 R15: 0000000000000001
FS: 0000000000000000(0000) GS:ffff8e60bfc00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f4f112cf118 CR3: 000000009f562000 CR4: 00000000000007f0
Call Trace:
[<ffffffff87228a5a>] __build_skb+0x2a/0x1b0
[<ffffffff87228d49>] __netdev_alloc_skb+0xc9/0x110
[<ffffffffc01455f6>] vmxnet3_rq_rx_complete+0x546/0xef0 [vmxnet3]
[<ffffffffc01460f6>] vmxnet3_poll_rx_only+0x36/0xa0 [vmxnet3]
[<ffffffff8723af1f>] net_rx_action+0x26f/0x390
[<ffffffff86ca1075>] __do_softirq+0xf5/0x280
[<ffffffff86ca1238>] run_ksoftirqd+0x38/0x50
[<ffffffff86cca734>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff86cca5f0>] ? lg_double_unlock+0x40/0x40
[<ffffffff86cc1da1>] kthread+0xd1/0xe0
[<ffffffff86cc1cd0>] ? insert_kthread_work+0x40/0x40
[<ffffffff87375c37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff86cc1cd0>] ? insert_kthread_work+0x40/0x40
Code: 4e 1f 79 49 8b 50 08 4d 8b 20 49 8b 40 10 4d 85 e4 0f 84 28 01 00 00 48 85 c0 0f 84 1f 01 00 00 49 63 46 20 48 8d 4a 01 4d 8b 06 <49> 8b 1c 04 4c 89 e0 65 49 0f c7 08 0f 94 c0 84 c0 74 ba 49 63
Complete report #274776
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
__netdev_alloc_skb
|
vmlinux |
0xffffffff81578164
Build id: 3.10.0-693.el7.x86_64
|
0xb4
|
|
2
|
vmxnet3_rq_rx_complete
|
vmxnet3 |
0xffffffffc00ca919
Build id: 3.10.0-693.el7.x86_64
|
0x199
|
|
3
|
vmxnet3_poll_rx_only
|
vmxnet3 |
0xffffffffc00cb50a
Build id: 3.10.0-693.el7.x86_64
|
0x3a
|
|
4
|
net_rx_action
|
vmlinux |
0xffffffff8158799d
Build id: 3.10.0-693.el7.x86_64
|
0x16d
|
|
5
|
__do_softirq
|
vmlinux |
0xffffffff81090b3f
Build id: 3.10.0-693.el7.x86_64
|
0xef
|
|
6
|
run_ksoftirqd
|
vmlinux |
0xffffffff81090d08
Build id: 3.10.0-693.el7.x86_64
|
0x38
|
|
7
|
smpboot_thread_fn
|
vmlinux |
0xffffffff810b909f
Build id: 3.10.0-693.el7.x86_64
|
0x12f
|
|
8
|
|
vmlinux |
0xffffffff810b8f70
Build id: 3.10.0-693.el7.x86_64
|
0x40
|
|
9
|
kthread
|
vmlinux |
0xffffffff810b098f
Build id: 3.10.0-693.el7.x86_64
|
0xcf
|
|
10
|
|
vmlinux |
0xffffffff810b08c0
Build id: 3.10.0-693.el7.x86_64
|
0x40
|
|
11
|
ret_from_fork
|
vmlinux |
0xffffffff816b4f18
Build id: 3.10.0-693.el7.x86_64
|
0x58
|
|
12
|
|
vmlinux |
0xffffffff810b08c0
Build id: 3.10.0-693.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 57s! [ksoftirqd/2:18]
Modules linked in: zram isofs vmw_vsock_vmci_transport vsock xfs libcrc32c sb_edac edac_core coretemp ppdev iosf_mbi crc32_pclmul ghash_clmulni_intel vmw_balloon aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr joydev sg parport_pc parport vmw_vmci shpchp i2c_piix4 nfsd auth_rpcgss nfs_acl lockd grace sunrpc binfmt_misc ip_tables ext4 mbcache jbd2 sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm crct10dif_pclmul crct10dif_common mptspi scsi_transport_spi mptscsih crc32c_intel mptbase serio_raw vmxnet3 ata_piix libata i2c_core floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 2 PID: 18 Comm: ksoftirqd/2 Tainted: G L ------------ 3.10.0-693.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/14/2014
task: ffff88017d68bf40 ti: ffff88017d6ac000 task.ti: ffff88017d6ac000
RIP: 0010:[<ffffffff81578164>] [<ffffffff81578164>] __netdev_alloc_skb+0xb4/0x110
RSP: 0018:ffff88017d6afcb0 EFLAGS: 00000202
RAX: ffff8800792af880 RBX: 0000000000000000 RCX: 0000000000007880
RDX: 0000000000008000 RSI: 0000000000007fff RDI: 0000000000000202
RBP: ffff88017d6afcd8 R08: ffffffff81911047 R09: ffffea0001e4ac00
R10: ffffea0001e4aa40 R11: ffff8801b7684360 R12: ffff8801bffd8000
R13: 0000000200000003 R14: 0000014100000000 R15: ffff8801bffd8008
FS: 0000000000000000(0000) GS:ffff8801bfd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007fecdd3f502c CR3: 0000000136dab000 CR4: 00000000000407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff8801b4f511b8 ffff8801b4f508c0 ffff8801b4f510c0 ffff8801a5b62b00
ffff8801af314240 ffff88017d6afd50 ffffffffc00ca919 ffff8801b4f510e0
ffff8801b7684348 ffff8801b8376240 ffff8801b7684360 0000000000000000
Call Trace:
[<ffffffffc00ca919>] vmxnet3_rq_rx_complete+0x199/0xc20 [vmxnet3]
[<ffffffffc00cb50a>] vmxnet3_poll_rx_only+0x3a/0xb0 [vmxnet3]
[<ffffffff8158799d>] net_rx_action+0x16d/0x380
[<ffffffff81090b3f>] __do_softirq+0xef/0x280
[<ffffffff81090d08>] run_ksoftirqd+0x38/0x50
[<ffffffff810b909f>] smpboot_thread_fn+0x12f/0x180
[<ffffffff810b8f70>] ? lg_double_unlock+0x40/0x40
[<ffffffff810b098f>] kthread+0xcf/0xe0
[<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
[<ffffffff816b4f18>] ret_from_fork+0x58/0x90
[<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
Code: 49 c7 c4 40 3d 01 00 65 4c 03 25 80 4f a9 7e 4c 89 e7 44 89 c2 44 89 ee e8 4a 4e c1 ff 45 0f b6 64 24 10 49 89 c7 4c 89 f7 57 9d <0f> 1f 44 00 00 4d 85 ff 74 4a 44 89 ee 4c 89 ff e8 e7 fc ff ff
Complete report #262585
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
__build_skb
|
vmlinux |
0xffffffff81577f3b
Build id: 3.10.0-693.el7.x86_64
|
0xdb
|
|
2
|
__netdev_alloc_skb
|
vmlinux |
0xffffffff81578179
Build id: 3.10.0-693.el7.x86_64
|
0xc9
|
|
3
|
vmxnet3_rq_rx_complete
|
vmxnet3 |
0xffffffffc006c919
Build id: 3.10.0-693.el7.x86_64
|
0x199
|
|
4
|
vmxnet3_poll_rx_only
|
vmxnet3 |
0xffffffffc006d50a
Build id: 3.10.0-693.el7.x86_64
|
0x3a
|
|
5
|
net_rx_action
|
vmlinux |
0xffffffff8158799d
Build id: 3.10.0-693.el7.x86_64
|
0x16d
|
|
6
|
__do_softirq
|
vmlinux |
0xffffffff81090b3f
Build id: 3.10.0-693.el7.x86_64
|
0xef
|
|
7
|
run_ksoftirqd
|
vmlinux |
0xffffffff81090d08
Build id: 3.10.0-693.el7.x86_64
|
0x38
|
|
8
|
smpboot_thread_fn
|
vmlinux |
0xffffffff810b909f
Build id: 3.10.0-693.el7.x86_64
|
0x12f
|
|
9
|
|
vmlinux |
0xffffffff810b8f70
Build id: 3.10.0-693.el7.x86_64
|
0x40
|
|
10
|
kthread
|
vmlinux |
0xffffffff810b098f
Build id: 3.10.0-693.el7.x86_64
|
0xcf
|
|
11
|
|
vmlinux |
0xffffffff810b08c0
Build id: 3.10.0-693.el7.x86_64
|
0x40
|
|
12
|
ret_from_fork
|
vmlinux |
0xffffffff816b4f18
Build id: 3.10.0-693.el7.x86_64
|
0x58
|
|
13
|
|
vmlinux |
0xffffffff810b08c0
Build id: 3.10.0-693.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#9 stuck for 22s! [ksoftirqd/9:55]
Modules linked in: vxlan ip6_udp_tunnel udp_tunnel iptable_mangle xt_mark veth xt_nat ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 xt_addrtype iptable_filter xt_conntrack nf_nat nf_conntrack br_netfilter bridge stp llc overlay(T) vmw_vsock_vmci_transport vboxpci(OE) vsock vboxnetadp(OE) vboxnetflt(OE) vboxdrv(OE) sb_edac edac_core coretemp iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd nfit ppdev pcspkr joydev vmw_balloon libnvdimm sg parport_pc parport vmw_vmci shpchp i2c_piix4 nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx sd_mod crc_t10dif crct10dif_generic drm_kms_helper
syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix libata crct10dif_pclmul crct10dif_common crc32c_intel serio_raw vmxnet3 i2c_core vmw_pvscsi floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 9 PID: 55 Comm: ksoftirqd/9 Tainted: G OEL ------------ T 3.10.0-693.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff8808b4ef9fa0 ti: ffff8808b4f08000 task.ti: ffff8808b4f08000
RIP: 0010:[<ffffffff81577f3b>] [<ffffffff81577f3b>] __build_skb+0xdb/0x1b0
RSP: 0018:ffff8808b4f0bc90 EFLAGS: 00010282
RAX: 00000000ffffffff RBX: ffff88103fcd9c40 RCX: 00000000ffffffff
RDX: ffff8807bacfff00 RSI: 0000000000000640 RDI: ffff880f07c11d40
RBP: ffff8808b4f0bca0 R08: 0000000000019c40 R09: 0000000000000000
R10: ffff881017b579c0 R11: ffff881016fe8ea0 R12: ffff8808b4f0bc20
R13: ffffffff81586f22 R14: ffff8808b4f0bc60 R15: 0000000000000008
FS: 0000000000000000(0000) GS:ffff88103fcc0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 0000555868283db8 CR3: 0000001018a18000 CR4: 00000000000407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff88081c43c000 0000000000000000 ffff8808b4f0bcd8 ffffffff81578179
ffff88081c43d378 ffff88081c43c8c0 ffff88081c43d280 ffff8810186a8300
ffff88101767c260 ffff8808b4f0bd50 ffffffffc006c919 ffff88081c43d2a0
Call Trace:
[<ffffffff81578179>] __netdev_alloc_skb+0xc9/0x110
[<ffffffffc006c919>] vmxnet3_rq_rx_complete+0x199/0xc20 [vmxnet3]
[<ffffffffc006d50a>] vmxnet3_poll_rx_only+0x3a/0xb0 [vmxnet3]
[<ffffffff8158799d>] net_rx_action+0x16d/0x380
[<ffffffff81090b3f>] __do_softirq+0xef/0x280
[<ffffffff81090d08>] run_ksoftirqd+0x38/0x50
[<ffffffff810b909f>] smpboot_thread_fn+0x12f/0x180
[<ffffffff810b8f70>] ? lg_double_unlock+0x40/0x40
[<ffffffff810b098f>] kthread+0xcf/0xe0
[<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
[<ffffffff816b4f18>] ret_from_fork+0x58/0x90
[<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
Code: 89 a2 e0 00 00 00 4c 89 a2 e8 00 00 00 c7 82 d8 00 00 00 00 00 00 00 89 b2 dc 00 00 00 66 89 82 c4 00 00 00 66 89 8a c0 00 00 00 <48> c7 07 00 00 00 00 48 c7 47 08 00 00 00 00 48 c7 47 10 00 00
Complete report #79927
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
__build_skb
|
vmlinux |
0xffffffffb1841d2b
Build id: 3.10.0-1127.13.1.el7.x86_64
|
0xdb
|
|
2
|
__netdev_alloc_skb
|
vmlinux |
0xffffffffb1841f69
Build id: 3.10.0-1127.13.1.el7.x86_64
|
0xc9
|
|
3
|
vmxnet3_rq_rx_complete
|
vmxnet3 |
0xffffffffc01de2de
Build id: 3.10.0-1127.13.1.el7.x86_64
|
0x54e
|
|
4
|
|
vmlinux |
0xffffffffb12e0000
Build id: 3.10.0-1127.13.1.el7.x86_64
|
0x50
|
|
5
|
vmxnet3_poll_rx_only
|
vmxnet3 |
0xffffffffc01df0e6
Build id: 3.10.0-1127.13.1.el7.x86_64
|
0x36
|
|
6
|
net_rx_action
|
vmlinux |
0xffffffffb18545bf
Build id: 3.10.0-1127.13.1.el7.x86_64
|
0x26f
|
|
7
|
__do_softirq
|
vmlinux |
0xffffffffb12a5695
Build id: 3.10.0-1127.13.1.el7.x86_64
|
0xf5
|
|
8
|
run_ksoftirqd
|
vmlinux |
0xffffffffb12a5858
Build id: 3.10.0-1127.13.1.el7.x86_64
|
0x38
|
|
9
|
smpboot_thread_fn
|
vmlinux |
0xffffffffb12cf0e4
Build id: 3.10.0-1127.13.1.el7.x86_64
|
0x144
|
|
10
|
|
vmlinux |
0xffffffffb12cefa0
Build id: 3.10.0-1127.13.1.el7.x86_64
|
0x40
|
|
11
|
kthread
|
vmlinux |
0xffffffffb12c6691
Build id: 3.10.0-1127.13.1.el7.x86_64
|
0xd1
|
|
12
|
|
vmlinux |
0xffffffffb12c65c0
Build id: 3.10.0-1127.13.1.el7.x86_64
|
0x40
|
|
13
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffb1992d37
Build id: 3.10.0-1127.13.1.el7.x86_64
|
0x21
|
|
14
|
|
vmlinux |
0xffffffffb12c65c0
Build id: 3.10.0-1127.13.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 23s! [ksoftirqd/2:19]
Modules linked in: xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netbios_ns nf_conntrack_broadcast ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter ip_tables tcp_lp fuse tun devlink bridge stp llc ip_set nfnetlink vmw_vsock_vmci_transport vsock sunrpc sb_edac coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper ppdev cryptd pcspkr vmw_balloon joydev sg parport_pc parport vmw_vmci
i2c_piix4 xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper ata_piix syscopyarea sysfillrect crct10dif_pclmul sysimgblt crct10dif_common fb_sys_fops ttm libata drm mptspi crc32c_intel scsi_transport_spi mptscsih serio_raw vmxnet3 mptbase drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod [last unloaded: ip_tables]
CPU: 2 PID: 19 Comm: ksoftirqd/2 Kdump: loaded Tainted: G L ------------ 3.10.0-1127.13.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff8c9cb9590000 ti: ffff8c9cb958c000 task.ti: ffff8c9cb958c000
RIP: 0010:[<ffffffffb1841d2b>] [<ffffffffb1841d2b>] __build_skb+0xdb/0x1b0
RSP: 0018:ffff8c9cb958fc88 EFLAGS: 00010286
RAX: 00000000ffffffff RBX: ffff8c9cb9590000 RCX: 00000000ffffffff
RDX: ffff8c9c33a70000 RSI: 0000000000000640 RDI: ffff8c9b9c8c7ec0
RBP: ffff8c9cb958fc98 R08: 000000000001f120 R09: 0000000000000000
R10: ffff8c9cbfbd4c60 R11: ffff8c9cb47648c0 R12: ffff8c9cb9590000
R13: ffff8c9cb9590000 R14: 0000000000000000 R15: 0000000000000000
FS: 0000000000000000(0000) GS:ffff8c9cbfd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fccadbb0000 CR3: 00000000b9270000 CR4: 00000000000607e0
Call Trace:
[<ffffffffb1841f69>] __netdev_alloc_skb+0xc9/0x110
[<ffffffffc01de2de>] vmxnet3_rq_rx_complete+0x54e/0xef0 [vmxnet3]
[<ffffffffb12e0000>] ? tg_unthrottle_up+0x50/0x50
[<ffffffffc01df0e6>] vmxnet3_poll_rx_only+0x36/0xa0 [vmxnet3]
[<ffffffffb18545bf>] net_rx_action+0x26f/0x390
[<ffffffffb12a5695>] __do_softirq+0xf5/0x280
[<ffffffffb12a5858>] run_ksoftirqd+0x38/0x50
[<ffffffffb12cf0e4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffb12cefa0>] ? lg_double_unlock+0x40/0x40
[<ffffffffb12c6691>] kthread+0xd1/0xe0
[<ffffffffb12c65c0>] ? insert_kthread_work+0x40/0x40
[<ffffffffb1992d37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffb12c65c0>] ? insert_kthread_work+0x40/0x40
Code: 89 a2 e0 00 00 00 4c 89 a2 e8 00 00 00 c7 82 d8 00 00 00 00 00 00 00 89 b2 dc 00 00 00 66 89 82 c4 00 00 00 66 89 8a c0 00 00 00 <48> c7 07 00 00 00 00 48 c7 47 08 00 00 00 00 48 c7 47 10 00 00
Complete report #66502
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
__build_skb
|
vmlinux |
0xffffffffa4229b0b
Build id: 3.10.0-957.27.2.el7.x86_64
|
0xdb
|
|
2
|
__netdev_alloc_skb
|
vmlinux |
0xffffffffa4229d49
Build id: 3.10.0-957.27.2.el7.x86_64
|
0xc9
|
|
3
|
vmxnet3_rq_rx_complete
|
vmxnet3 |
0xffffffffc016e5f6
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x546
|
|
4
|
vmxnet3_poll_rx_only
|
vmxnet3 |
0xffffffffc016f0f6
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x36
|
|
5
|
net_rx_action
|
vmlinux |
0xffffffffa423bf1f
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x26f
|
|
6
|
__do_softirq
|
vmlinux |
0xffffffffa3ca2155
Build id: 3.10.0-957.27.2.el7.x86_64
|
0xf5
|
|
7
|
run_ksoftirqd
|
vmlinux |
0xffffffffa3ca2318
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x38
|
|
8
|
smpboot_thread_fn
|
vmlinux |
0xffffffffa3ccb814
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x144
|
|
9
|
|
vmlinux |
0xffffffffa3ccb6d0
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x40
|
|
10
|
kthread
|
vmlinux |
0xffffffffa3cc2e81
Build id: 3.10.0-957.27.2.el7.x86_64
|
0xd1
|
|
11
|
|
vmlinux |
0xffffffffa3cc2db0
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x40
|
|
12
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffa4376c37
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x21
|
|
13
|
|
vmlinux |
0xffffffffa3cc2db0
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 298s! [ksoftirqd/4:29]
Modules linked in: xt_multiport iptable_raw ip_set_hash_ip ip_set_hash_net ipip tunnel4 ip_tunnel xt_CHECKSUM iptable_mangle ipt_REJECT nf_reject_ipv4 tun ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink xt_conntrack ebtable_filter ebtables ip6table_filter ip6_tables devlink xt_addrtype xt_set xt_mark ip_set_hash_ipportnet ip_set_hash_ipportip ip_set_bitmap_port ip_set_hash_ipport ip_set nfnetlink dummy xt_comment iptable_nat nf_nat_ipv4 nf_nat overlay(T) nf_conntrack_ipv4 nf_defrag_ipv4 ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs nf_conntrack iptable_filter vmw_vsock_vmci_transport vsock sunrpc sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev vmw_balloon joydev pcspkr sg i2c_piix4 parport_pc parport vmw_vmci br_netfilter
bridge stp llc ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm mptspi ata_piix crct10dif_pclmul crct10dif_common scsi_transport_spi crc32c_intel mptscsih libata serio_raw nfit mptbase libnvdimm vmxnet3 drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 4 PID: 29 Comm: ksoftirqd/4 Tainted: G W L ------------ T 3.10.0-957.27.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff964539c70000 ti: ffff964539c78000 task.ti: ffff964539c78000
RIP: 0010:[<ffffffffa4229b0b>] [<ffffffffa4229b0b>] __build_skb+0xdb/0x1b0
RSP: 0018:ffff964539c7bc88 EFLAGS: 00010286
RAX: 00000000ffffffff RBX: ffff9647fffdb008 RCX: 00000000ffffffff
RDX: ffff9647ee48a500 RSI: 0000000000000640 RDI: ffff964401e2fec0
RBP: ffff964539c7bc98 R08: 000000000001f120 R09: 0000000000000000
R10: ffff9647ffbd6100 R11: ffff9647e915c8c0 R12: ffff964539c70000
R13: ffff964539c70000 R14: ffff964539c70000 R15: ffff9647e915e478
FS: 0000000000000000(0000) GS:ffff9647ffd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f033a9db000 CR3: 0000000035fa2000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffa4229d49>] __netdev_alloc_skb+0xc9/0x110
[<ffffffffc016e5f6>] vmxnet3_rq_rx_complete+0x546/0xef0 [vmxnet3]
[<ffffffffc016f0f6>] vmxnet3_poll_rx_only+0x36/0xa0 [vmxnet3]
[<ffffffffa423bf1f>] net_rx_action+0x26f/0x390
[<ffffffffa3ca2155>] __do_softirq+0xf5/0x280
[<ffffffffa3ca2318>] run_ksoftirqd+0x38/0x50
[<ffffffffa3ccb814>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffa3ccb6d0>] ? lg_double_unlock+0x40/0x40
[<ffffffffa3cc2e81>] kthread+0xd1/0xe0
[<ffffffffa3cc2db0>] ? insert_kthread_work+0x40/0x40
[<ffffffffa4376c37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffa3cc2db0>] ? insert_kthread_work+0x40/0x40
Code: 89 a2 e0 00 00 00 4c 89 a2 e8 00 00 00 c7 82 d8 00 00 00 00 00 00 00 89 b2 dc 00 00 00 66 89 82 c4 00 00 00 66 89 8a c0 00 00 00 <48> c7 07 00 00 00 00 48 c7 47 08 00 00 00 00 48 c7 47 10 00 00
Complete report #16495
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
__netdev_alloc_skb
|
vmlinux |
0xffffffff82241f54
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xb4
|
|
2
|
vmxnet3_rq_rx_complete
|
vmxnet3 |
0xffffffffc03042de
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x54e
|
|
3
|
vmxnet3_poll_rx_only
|
vmxnet3 |
0xffffffffc03050e6
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x36
|
|
4
|
net_rx_action
|
vmlinux |
0xffffffff822545bf
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x26f
|
|
5
|
__do_softirq
|
vmlinux |
0xffffffff81ca5695
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xf5
|
|
6
|
run_ksoftirqd
|
vmlinux |
0xffffffff81ca5858
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x38
|
|
7
|
smpboot_thread_fn
|
vmlinux |
0xffffffff81ccf0e4
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x144
|
|
8
|
|
vmlinux |
0xffffffff81ccefa0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x40
|
|
9
|
kthread
|
vmlinux |
0xffffffff81cc6691
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xd1
|
|
10
|
|
vmlinux |
0xffffffff81cc65c0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x40
|
|
11
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff82392d1d
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x7
|
|
12
|
|
vmlinux |
0xffffffff81cc65c0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#22 stuck for 22s! [ksoftirqd/22:120]
Modules linked in: tcp_lp fuse veeamsnap(OE) veth binfmt_misc nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter nls_utf8 cifs ccm dns_resolver overlay(T) vmw_vsock_vmci_transport vsock ppdev vmw_balloon iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr sg vmw_vmci i2c_piix4 parport_pc parport nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx sd_mod crc_t10dif crct10dif_generic drm_kms_helper
syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci nfit ata_piix libahci drm crct10dif_pclmul libnvdimm crct10dif_common libata crc32c_intel serio_raw vmxnet3 vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 22 PID: 120 Comm: ksoftirqd/22 Kdump: loaded Tainted: G OEL ------------ T 3.10.0-1127.19.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/28/2017
task: ffff9be093fca0e0 ti: ffff9be093fe8000 task.ti: ffff9be093fe8000
RIP: 0010:[<ffffffff82241f54>] [<ffffffff82241f54>] __netdev_alloc_skb+0xb4/0x110
RSP: 0000:ffff9be093febca8 EFLAGS: 00000202
RAX: ffff9bf6b45cf880 RBX: ffff9bfebffdb000 RCX: 0000000000007880
RDX: 0000000000008000 RSI: 0000000000007fff RDI: 0000000000000202
RBP: ffff9be093febcd0 R08: ffffffff8268d58f R09: fffff7589dd17400
R10: ffff9bfebffcefc0 R11: ffff9c1d7d9d48c0 R12: 0000000300000000
R13: ffff9bfebffdb008 R14: ffff9be093fca0e0 R15: ffff9be093fca0e0
FS: 0000000000000000(0000) GS:ffff9bfe42980000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f489c0cab20 CR3: 000000240c210000 CR4: 00000000007607e0
DR0: 00000000006050d4 DR1: 00007fffffffe908 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000600
PKRU: 00000000
Call Trace:
[<ffffffffc03042de>] vmxnet3_rq_rx_complete+0x54e/0xef0 [vmxnet3]
[<ffffffffc03050e6>] vmxnet3_poll_rx_only+0x36/0xa0 [vmxnet3]
[<ffffffff822545bf>] net_rx_action+0x26f/0x390
[<ffffffff81ca5695>] __do_softirq+0xf5/0x280
[<ffffffff81ca5858>] run_ksoftirqd+0x38/0x50
[<ffffffff81ccf0e4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff81ccefa0>] ? lg_double_unlock+0x40/0x40
[<ffffffff81cc6691>] kthread+0xd1/0xe0
[<ffffffff81cc65c0>] ? insert_kthread_work+0x40/0x40
[<ffffffff82392d1d>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffff81cc65c0>] ? insert_kthread_work+0x40/0x40
Code: 49 c7 c4 a0 9f 01 00 65 4c 03 25 a8 f2 dc 7d 4c 89 e7 44 89 c2 44 89 ee e8 7a 73 b8 ff 45 0f b6 64 24 10 49 89 c7 4c 89 f7 57 9d <0f> 1f 44 00 00 4d 85 ff 74 4a 44 89 ee 4c 89 ff e8 e7 fc ff ff
Complete report #8125
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
__build_skb
|
vmlinux |
0xffffffff86a28b0b
Build id: 3.10.0-957.21.3.el7.x86_64
|
0xdb
|
|
2
|
__netdev_alloc_skb
|
vmlinux |
0xffffffff86a28d49
Build id: 3.10.0-957.21.3.el7.x86_64
|
0xc9
|
|
3
|
vmxnet3_rq_rx_complete
|
vmxnet3 |
0xffffffffc031b5f6
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x546
|
|
4
|
|
vmlinux |
0xffffffff864e0000
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x40
|
|
5
|
vmxnet3_poll_rx_only
|
vmxnet3 |
0xffffffffc031c0f6
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x36
|
|
6
|
net_rx_action
|
vmlinux |
0xffffffff86a3af1f
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x26f
|
|
7
|
__do_softirq
|
vmlinux |
0xffffffff864a1075
Build id: 3.10.0-957.21.3.el7.x86_64
|
0xf5
|
|
8
|
run_ksoftirqd
|
vmlinux |
0xffffffff864a1238
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x38
|
|
9
|
smpboot_thread_fn
|
vmlinux |
0xffffffff864ca734
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x144
|
|
10
|
|
vmlinux |
0xffffffff864ca5f0
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x40
|
|
11
|
kthread
|
vmlinux |
0xffffffff864c1da1
Build id: 3.10.0-957.21.3.el7.x86_64
|
0xd1
|
|
12
|
|
vmlinux |
0xffffffff864c1cd0
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x40
|
|
13
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff86b75c37
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x21
|
|
14
|
|
vmlinux |
0xffffffff864c1cd0
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 96s! [ksoftirqd/1:14]
Modules linked in: xt_addrtype br_netfilter xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter overlay(T) vmw_vsock_vmci_transport vsock sunrpc coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ppdev ablk_helper cryptd vmw_balloon joydev pcspkr sg vmw_vmci parport_pc parport i2c_piix4 ip_tables xfs libcrc32c sr_mod
cdrom sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel serio_raw ata_generic pata_acpi floppy vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm mptspi libata scsi_transport_spi vmxnet3 mptscsih drm_panel_orientation_quirks mptbase dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 14 Comm: ksoftirqd/1 Kdump: loaded Tainted: G L ------------ T 3.10.0-957.21.3.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff9155b9550000 ti: ffff9155b954c000 task.ti: ffff9155b954c000
RIP: 0010:[<ffffffff86a28b0b>] [<ffffffff86a28b0b>] __build_skb+0xdb/0x1b0
RSP: 0018:ffff9155b954fc88 EFLAGS: 00010282
RAX: 00000000ffffffff RBX: ffffffff871618d0 RCX: 00000000ffffffff
RDX: ffff915531c73700 RSI: 0000000000000640 RDI: ffff915503ef5940
RBP: ffff9155b954fc98 R08: 000000000001f120 R09: 0000000000000000
R10: ffff915532b10de0 R11: ffff9155b44dc8c0 R12: ffffffff86a3a579
R13: ffff9155b954fc58 R14: 000000000000dd86 R15: ffff9155b44dc000
FS: 0000000000000000(0000) GS:ffff9155bfd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f0d3416f000 CR3: 00000000b65ea000 CR4: 00000000001607e0
Call Trace:
[<ffffffff86a28d49>] __netdev_alloc_skb+0xc9/0x110
[<ffffffffc031b5f6>] vmxnet3_rq_rx_complete+0x546/0xef0 [vmxnet3]
[<ffffffff864e0000>] ? init_cfs_bandwidth+0x40/0x90
[<ffffffffc031c0f6>] vmxnet3_poll_rx_only+0x36/0xa0 [vmxnet3]
[<ffffffff86a3af1f>] net_rx_action+0x26f/0x390
[<ffffffff864a1075>] __do_softirq+0xf5/0x280
[<ffffffff864a1238>] run_ksoftirqd+0x38/0x50
[<ffffffff864ca734>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff864ca5f0>] ? lg_double_unlock+0x40/0x40
[<ffffffff864c1da1>] kthread+0xd1/0xe0
[<ffffffff864c1cd0>] ? insert_kthread_work+0x40/0x40
[<ffffffff86b75c37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff864c1cd0>] ? insert_kthread_work+0x40/0x40
Code: 89 a2 e0 00 00 00 4c 89 a2 e8 00 00 00 c7 82 d8 00 00 00 00 00 00 00 89 b2 dc 00 00 00 66 89 82 c4 00 00 00 66 89 8a c0 00 00 00 <48> c7 07 00 00 00 00 48 c7 47 08 00 00 00 00 48 c7 47 10 00 00
Complete report #7537
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
__netdev_alloc_skb
|
vmlinux |
0xffffffffadc3df34
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0xb4
|
|
2
|
vmxnet3_rq_rx_complete
|
vmxnet3 |
0xffffffffc010e2de
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x54e
|
|
3
|
vmxnet3_poll_rx_only
|
vmxnet3 |
0xffffffffc010f0e6
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x36
|
|
4
|
net_rx_action
|
vmlinux |
0xffffffffadc5057f
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x26f
|
|
5
|
__do_softirq
|
vmlinux |
0xffffffffad6a5435
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0xf5
|
|
6
|
run_ksoftirqd
|
vmlinux |
0xffffffffad6a55f8
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x38
|
|
7
|
smpboot_thread_fn
|
vmlinux |
0xffffffffad6ced54
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x144
|
|
8
|
|
vmlinux |
0xffffffffad6cec10
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x40
|
|
9
|
kthread
|
vmlinux |
0xffffffffad6c6321
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0xd1
|
|
10
|
|
vmlinux |
0xffffffffad6c6250
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x40
|
|
11
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffadd8dd37
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x21
|
|
12
|
|
vmlinux |
0xffffffffad6c6250
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#7 stuck for 21s! [ksoftirqd/7:44]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter devlink ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod ppdev sb_edac vmw_balloon coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr joydev sg parport_pc vmw_vmci parport i2c_piix4 binfmt_misc ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm mptspi drm ata_piix scsi_transport_spi mptscsih crct10dif_pclmul
libata crct10dif_common crc32c_intel serio_raw mptbase vmxnet3 drm_panel_orientation_quirks floppy
CPU: 7 PID: 44 Comm: ksoftirqd/7 Kdump: loaded Tainted: G L ------------ 3.10.0-1062.18.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/30/2014
task: ffff9820eeec1070 ti: ffff9820eeecc000 task.ti: ffff9820eeecc000
RIP: 0010:[<ffffffffadc3df34>] [<ffffffffadc3df34>] __netdev_alloc_skb+0xb4/0x110
RSP: 0018:ffff9820eeecfca8 EFLAGS: 00000202
RAX: ffff982378c1f880 RBX: ffff982a3ffdb000 RCX: 0000000000007880
RDX: 0000000000008000 RSI: 0000000000007fff RDI: 0000000000000202
RBP: ffff9820eeecfcd0 R08: ffffffffae08b1ff R09: fffff6464fe30800
R10: ffff982a3fbd44e0 R11: ffff982a0f0dc8c0 R12: 0000000300000000
R13: ffff982a3ffdb008 R14: ffff9820eeec1070 R15: ffff9820eeec1070
FS: 0000000000000000(0000) GS:ffff982a3fdc0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fb687562000 CR3: 00000009e8078000 CR4: 00000000001607e0
Call Trace:
[<ffffffffc010e2de>] vmxnet3_rq_rx_complete+0x54e/0xef0 [vmxnet3]
[<ffffffffc010f0e6>] vmxnet3_poll_rx_only+0x36/0xa0 [vmxnet3]
[<ffffffffadc5057f>] net_rx_action+0x26f/0x390
[<ffffffffad6a5435>] __do_softirq+0xf5/0x280
[<ffffffffad6a55f8>] run_ksoftirqd+0x38/0x50
[<ffffffffad6ced54>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffad6cec10>] ? lg_double_unlock+0x40/0x40
[<ffffffffad6c6321>] kthread+0xd1/0xe0
[<ffffffffad6c6250>] ? insert_kthread_work+0x40/0x40
[<ffffffffadd8dd37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffad6c6250>] ? insert_kthread_work+0x40/0x40
Code: 49 c7 c4 60 9f 01 00 65 4c 03 25 88 32 3d 52 4c 89 e7 44 89 c2 44 89 ee e8 da a7 b8 ff 45 0f b6 64 24 10 49 89 c7 4c 89 f7 57 9d <0f> 1f 44 00 00 4d 85 ff 74 4a 44 89 ee 4c 89 ff e8 e7 fc ff ff
Complete report #3888
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
__build_skb
|
vmlinux |
0xffffffff81577f3b
Build id: 3.10.0-693.2.2.el7.x86_64
|
0xdb
|
|
2
|
__netdev_alloc_skb
|
vmlinux |
0xffffffff81578179
Build id: 3.10.0-693.2.2.el7.x86_64
|
0xc9
|
|
3
|
vmxnet3_rq_rx_complete
|
vmxnet3 |
0xffffffffc003d919
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x199
|
|
4
|
vmxnet3_poll_rx_only
|
vmxnet3 |
0xffffffffc003e50a
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x3a
|
|
5
|
net_rx_action
|
vmlinux |
0xffffffff8158799d
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x16d
|
|
6
|
__do_softirq
|
vmlinux |
0xffffffff81090b3f
Build id: 3.10.0-693.2.2.el7.x86_64
|
0xef
|
|
7
|
run_ksoftirqd
|
vmlinux |
0xffffffff81090d08
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x38
|
|
8
|
smpboot_thread_fn
|
vmlinux |
0xffffffff810b909f
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x12f
|
|
9
|
|
vmlinux |
0xffffffff810b8f70
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x40
|
|
10
|
kthread
|
vmlinux |
0xffffffff810b098f
Build id: 3.10.0-693.2.2.el7.x86_64
|
0xcf
|
|
11
|
|
vmlinux |
0xffffffff810b08c0
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x40
|
|
12
|
ret_from_fork
|
vmlinux |
0xffffffff816b4f58
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x58
|
|
13
|
|
vmlinux |
0xffffffff810b08c0
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 21s! [ksoftirqd/0:3]
Modules linked in: ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sb_edac edac_core coretemp iosf_mbi crc32_pclmul ppdev ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper vmw_balloon cryptd sg joydev pcspkr parport_pc shpchp parport i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper
syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix mptspi scsi_transport_spi mptscsih mptbase drm crct10dif_pclmul crct10dif_common libata crc32c_intel serio_raw vmxnet3 i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 3 Comm: ksoftirqd/0 Tainted: G L ------------ 3.10.0-693.2.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff8801542d9fa0 ti: ffff880154370000 task.ti: ffff880154370000
RIP: 0010:[<ffffffff81577f3b>] [<ffffffff81577f3b>] __build_skb+0xdb/0x1b0
RSP: 0018:ffff880154373c90 EFLAGS: 00010282
RAX: 00000000ffffffff RBX: ffff881f8fc19c40 RCX: 00000000ffffffff
RDX: ffff880a9b690c00 RSI: 0000000000000640 RDI: ffff880c1a9b1d40
RBP: ffff880154373ca0 R08: 0000000000019c40 R09: 0000000000000000
R10: ffff881f01639590 R11: ffff881f0069c858 R12: ffffffff81586f22
R13: ffff880154373c68 R14: 0000000000000608 R15: ffffffff81b257f8
FS: 0000000000000000(0000) GS:ffff881f8fc00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f917758d000 CR3: 0000001efc6fd000 CR4: 00000000003407f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffff881f0e280000 0000000000000000 ffff880154373cd8 ffffffff81578179
ffff881f0e2811b8 ffff881f0e2808c0 ffff881f0e2810c0 ffff880a9fb4b800
ffff881f03a42870 ffff880154373d50 ffffffffc003d919 ffff881f0e2810e0
Call Trace:
[<ffffffff81578179>] __netdev_alloc_skb+0xc9/0x110
[<ffffffffc003d919>] vmxnet3_rq_rx_complete+0x199/0xc20 [vmxnet3]
[<ffffffffc003e50a>] vmxnet3_poll_rx_only+0x3a/0xb0 [vmxnet3]
[<ffffffff8158799d>] net_rx_action+0x16d/0x380
[<ffffffff81090b3f>] __do_softirq+0xef/0x280
[<ffffffff81090d08>] run_ksoftirqd+0x38/0x50
[<ffffffff810b909f>] smpboot_thread_fn+0x12f/0x180
[<ffffffff810b8f70>] ? lg_double_unlock+0x40/0x40
[<ffffffff810b098f>] kthread+0xcf/0xe0
[<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
[<ffffffff816b4f58>] ret_from_fork+0x58/0x90
[<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
Code: 89 a2 e0 00 00 00 4c 89 a2 e8 00 00 00 c7 82 d8 00 00 00 00 00 00 00 89 b2 dc 00 00 00 66 89 82 c4 00 00 00 66 89 8a c0 00 00 00 <48> c7 07 00 00 00 00 48 c7 47 08 00 00 00 00 48 c7 47 10 00 00