BUG: soft lockup - CPU#0 stuck for 33s! [nagios:23793]
Modules linked in: snapapi26(POE) fuse iptable_nat nf_nat_ipv4 nf_nat ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 vmw_vsock_vmci_transport vsock xt_conntrack nf_conntrack iptable_filter coretemp crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev pcspkr vmw_balloon sg shpchp i2c_piix4 parport_pc parport vmw_vmci nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel vmwgfx serio_raw drm_kms_helper ata_piix ttm libata e1000 drm mptspi scsi_transport_spi mptscsih mptbase i2c_core floppy dm_mirror dm_region_hash dm_log dm_mod [last unloaded: edd]
CPU: 0 PID: 23793 Comm: nagios Tainted: P           OEL ------------   3.10.0-327.36.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013
task: ffff88001893a280 ti: ffff8800370b4000 task.ti: ffff8800370b4000
RIP: 0010:[<ffffffffa00663be>]  [<ffffffffa00663be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff8800370b7b78  EFLAGS: 00000246
RAX: ffffc90008340000 RBX: ffff88003b413380 RCX: 0000000000000018
RDX: ffff8800361dfe00 RSI: ffff8800392dc000 RDI: 000000000000000e
RBP: ffff8800370b7b88 R08: 0000000000000003 R09: 000000000e948fff
R10: ffff88003887ac40 R11: ffff88003887ac40 R12: ffff88002415aa80
R13: ffff8800370b7aec R14: ffffffff812d08fb R15: ffff8800370b7b30
FS:  00007fb317b1d740(0000) GS:ffff88003fc00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fb317b37000 CR3: 000000003b711000 CR4: 00000000000407f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff8800392dcd30 0000000000000040 ffff8800370b7c30 ffffffffa0088729
ffff8800392dc008 04000000370b7bf0 0000000000000018 ffff88003887ac40
0000000000000060 ffff8800392dc188 0000006000000018 ffff88000000000b
Call Trace:
[<ffffffffa0088729>] mptscsih_qcmd+0x249/0x820 [mptscsih]
[<ffffffffa00812b0>] mptspi_qcmd+0x50/0xe0 [mptspi]
[<ffffffff81417f4a>] scsi_dispatch_cmd+0xaa/0x230
[<ffffffff81420ed1>] scsi_request_fn+0x501/0x770
[<ffffffff812c75b3>] __blk_run_queue+0x33/0x40
[<ffffffff812c766a>] queue_unplugged+0x2a/0xa0
[<ffffffff812cbee8>] blk_flush_plug_list+0x1d8/0x230
[<ffffffff812cc2f4>] blk_finish_plug+0x14/0x40
[<ffffffff81174c98>] generic_writepages+0x58/0x80
[<ffffffffa024d9b3>] xfs_vm_writepages+0x43/0x50 [xfs]
[<ffffffff81175d3e>] do_writepages+0x1e/0x40
[<ffffffff8116aad5>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffff8116abea>] filemap_write_and_wait_range+0x2a/0x70
[<ffffffffa02582a6>] xfs_file_fsync+0x66/0x200 [xfs]
[<ffffffff81210155>] do_fsync+0x65/0xa0
[<ffffffff81210420>] SyS_fsync+0x10/0x20
[<ffffffff81646a09>] system_call_fastpath+0x16/0x1b
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 43 07 

BUG: soft lockup - CPU#15 stuck for 28s! [sendmail:50642]
Modules linked in: binfmt_misc tun ebtable_filter ebtables ip6table_filter ip6_tables bridge stp llc dahdi(OF) crc_ccitt netatop(OF) xt_CHECKSUM iptable_mangle xt_nat ipt_MASQUERADE nf_nat_masquerade_ipv4 xt_addrtype iptable_nat nf_nat_ipv4 nf_nat nf_log_ipv4 nf_log_common xt_LOG ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_recent xt_multiport vmw_vsock_vmci_transport vsock xt_mark xt_conntrack nf_conntrack iptable_filter coretemp ext4 mbcache jbd2 snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus crc32_pclmul snd_seq ghash_clmulni_intel snd_seq_device snd_pcm ppdev aesni_intel lrw gf128mul vmw_balloon glue_helper snd_timer ablk_helper cryptd snd soundcore pcspkr ahci libahci sg i2c_piix4 vmw_vmci shpchp parport_pc parport nfsd auth_rpcgss nfs_acl lockd sunrpc ip_tables xfs libcrc32c
sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper crct10dif_pclmul crct10dif_common ttm crc32c_intel ata_piix mptspi drm serio_raw scsi_transport_spi libata mptscsih e1000 mptbase i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 15 PID: 50642 Comm: sendmail Tainted: GF          O--------------   3.10.0-229.14.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
task: ffff88032a6338e0 ti: ffff8801eb4a4000 task.ti: ffff8801eb4a4000
RIP: 0010:[<ffffffffa004a3be>]  [<ffffffffa004a3be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff8801eb4a7b78  EFLAGS: 00000246
RAX: ffffc900098c0000 RBX: ffff880328828c00 RCX: 0000000000000004
RDX: ffff88032c2d7c00 RSI: ffff88032c77a000 RDI: 000000000000000e
RBP: ffff8801eb4a7b88 R08: 0000000000000002 R09: 0000000000000000
R10: ffff880324408540 R11: 0000000000000000 R12: ffffffff812b310b
R13: ffff8801eb4a7b40 R14: 0000000000000001 R15: ffff8800353c4e18
FS:  00007fdfa0dda840(0000) GS:ffff8803327e0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ffc92ef3620 CR3: 000000032917b000 CR4: 00000000001407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff88032c77ad30 000000000000003c ffff8801eb4a7c30 ffffffffa00fc729
ffff88032c77a008 04000000067d37b8 0000000000000004 ffff880324408540
0000000000000010 ffff88032c77a188 0000006000000004 ffff8800357c29b0
Call Trace:
[<ffffffffa00fc729>] mptscsih_qcmd+0x249/0x820 [mptscsih]
[<ffffffffa00432b0>] mptspi_qcmd+0x50/0xe0 [mptspi]
[<ffffffff813f0539>] scsi_dispatch_cmd+0xb9/0x360
[<ffffffff813f969f>] scsi_request_fn+0x32f/0x570
[<ffffffff812aaa93>] __blk_run_queue+0x33/0x40
[<ffffffff812aab4a>] queue_unplugged+0x2a/0xa0
[<ffffffff812ae948>] blk_flush_plug_list+0x1d8/0x230
[<ffffffff812aed44>] blk_finish_plug+0x14/0x40
[<ffffffff81161e98>] generic_writepages+0x58/0x80
[<ffffffffa02a5ea3>] xfs_vm_writepages+0x43/0x50 [xfs]
[<ffffffff81162f3e>] do_writepages+0x1e/0x40
[<ffffffff81157db5>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffff81157eca>] filemap_write_and_wait_range+0x2a/0x70
[<ffffffffa02b0706>] xfs_file_fsync+0x66/0x1f0 [xfs]
[<ffffffff811f6ee5>] do_fsync+0x65/0xa0
[<ffffffff811c73e8>] ? SyS_write+0x58/0xb0
[<ffffffff811f71b0>] SyS_fsync+0x10/0x20
[<ffffffff81614389>] system_call_fastpath+0x16/0x1b
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 83 05 

NMI watchdog: BUG: soft lockup - CPU#1 stuck for 23s! [xfsaild/dm-0:548]
NMI watchdog: BUG: soft lockup - CPU#7 stuck for 23s! [bounce:6347]
Modules linked in: nfnetlink_queue nfnetlink_log nfnetlink bluetooth rfkill vmw_vsock_vmci_transport vsock intel_powerclamp coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev sg vmw_balloon pcspkr parport_pc parport shpchp i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sr_mod cdrom ata_generic sd_mod crc_t10dif crct10dif_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect mptspi sysimgblt scsi_transport_spi fb_sys_fops ttm crct10dif_pclmul crct10dif_common ata_piix drm crc32c_intel serio_raw mptscsih libata mptbase vmxnet3 i2c_core floppy fjes dm_mirror dm_region_hash dm_log dm_mod
CPU: 7 PID: 6347 Comm: bounce Tainted: G             L ------------   3.10.0-514.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/30/2014
task: ffff88080e3b5e20 ti: ffff880830b54000 task.ti: ffff880830b54000
RIP: 0010:[<ffffffffa00773be>]  [<ffffffffa00773be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff880830b57b78  EFLAGS: 00000246
RAX: ffffc90003500000 RBX: ffff8800359e58d0 RCX: 000000000000000b
RDX: ffff8806373d2600 RSI: ffff8804c503e000 RDI: 000000000000000e
RBP: ffff880830b57b88 R08: 0000000000000003 R09: ffff8806c50320d8
R10: ffff880436e80a80 R11: ffff880436e80a80 R12: ffff880434692100
R13: ffff880830b57aec R14: ffffffff812f4a0b R15: ffff880830b57b30
FS:  00007f2e29a75840(0000) GS:ffff880437d80000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f2e2b4c0fc8 CR3: 0000000433a4a000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffff8804c503ed30 0000000000000048 ffff880830b57c30 ffffffffa004f729
ffff8804c503e008 0400000030b57bf0 000000000000000b ffff880436e80a80
000000000000002c ffff8804c503e188 000000600000000b ffff880636082c50
Call Trace:
[<ffffffffa004f729>] mptscsih_qcmd+0x249/0x820 [mptscsih]
[<ffffffffa00702b0>] mptspi_qcmd+0x50/0xe0 [mptspi]
[<ffffffff814518da>] scsi_dispatch_cmd+0xaa/0x230
[<ffffffff8145a9e1>] scsi_request_fn+0x501/0x770
[<ffffffff812eb053>] __blk_run_queue+0x33/0x40
[<ffffffff812eb10a>] queue_unplugged+0x2a/0xa0
[<ffffffff812eff68>] blk_flush_plug_list+0x1d8/0x230
[<ffffffff812f0374>] blk_finish_plug+0x14/0x40
[<ffffffff8118c1a8>] generic_writepages+0x58/0x80
[<ffffffffa02a1063>] xfs_vm_writepages+0x53/0x90 [xfs]
[<ffffffff8118d24e>] do_writepages+0x1e/0x40
[<ffffffff81181e35>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffff81181f81>] filemap_write_and_wait_range+0x41/0x90
[<ffffffffa02ac846>] xfs_file_fsync+0x66/0x1e0 [xfs]
[<ffffffff8122fdb5>] do_fsync+0x65/0xa0
[<ffffffff81230080>] SyS_fsync+0x10/0x20
[<ffffffff816965c9>] system_call_fastpath+0x16/0x1b
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 53 08