| Operating system | Count |
|---|---|
| CentOS 7 | 10 |
| Architecture | Count |
|---|---|
| x86_64 | 10 |
| Related packages | Count |
|---|---|
| kernel | 10 |
Packages names with count less than the total count of reports are most probably not the cause of the problem.
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [dlm_i_cb/2:10024] Modules linked in: nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter tcp_lp fuse squashfs loop overlay(T) rctld(OE) parastor(OE) dlm(OE) kfnccli(OE) knal(OE) mmfs26(OE) mmfslinux(OE) tracedev(OE) rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_uverbs(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) skx_edac edac_core coretemp iTCO_wdt intel_rapl iTCO_vendor_support iosf_mbi kvm_intel kvm irqbypass sg joydev pcspkr mei_me shpchp lpc_ich mei i2c_i801 ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_cpufreq binfmt_misc knem(OE) nfsd auth_rpcgss nfs_acl lockd grace ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx5_ib(OE) ib_core(OE) 8021q garp stp llc mrp crct10dif_pclmul crct10dif_common crc32_pclmul ixgbe crc32c_intel ghash_clmulni_intel i2c_algo_bit drm_kms_helper aesni_intel lrw gf128mul glue_helper ablk_helper cryptd syscopyarea sysfillrect sysimgblt fb_sys_fops ttm dca drm mdio ahci libahci i40e scsi_transport_iscsi libata mpt3sas mlx5_core(OE) i2c_core mlxfw(OE) mlx_compat(OE) devlink ptp raid_class pps_core scsi_transport_sas wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod CPU: 0 PID: 10024 Comm: dlm_i_cb/2 Tainted: G W OEL ------------ T 3.10.0-693.el7.x86_64 #1 Hardware name: Sugon I620-G30/60P24-US, BIOS 0JGST031 08/02/2018 task: ffff8816f16b9fa0 ti: ffff8816410a8000 task.ti: ffff8816410a8000 RIP: 0010:[<ffffffffc19a4d27>] [<ffffffffc19a4d27>] fsc_shrink_dcache_parent+0xb7/0x180 [parastor] RSP: 0018:ffff8816410ab7e0 EFLAGS: 00000287 RAX: ffff880f0be99820 RBX: ffff881000472500 RCX: dead000000000200 RDX: ffff880f0be99800 RSI: ffff8816e172c290 RDI: ffff880f0be99800 RBP: ffff8816410ab850 R08: ffff880f0be99800 R09: 0000000000000002 R10: ffff881000472500 R11: ffff881028c42310 R12: ffff881000472500 R13: ffff881028c42310 R14: ffff8816e172c290 R15: ffff8816410ab7c0 FS: 0000000000000000(0000) GS:ffff880c0f600000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 00000000020a8008 CR3: 00000000019f2000 CR4: 00000000003407f0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 Stack: ffff880e11bd6500 ffff881626590000 ffff8816f16b9fa0 ffff881000472520 ffff8816e172c290 ffff8816e172c1b8 00000000ffffff10 ffff881000472500 0000000000000010 000000000000d580 0000000000000d3a ffff880fe6d21540 Call Trace: [<ffffffffc19aff5c>] fscex_drop_negative_dentries+0x86c/0x910 [parastor] [<ffffffffc19bc495>] ? __truncate_inode_blks_range+0xc5/0xe0 [parastor] [<ffffffffc1a25c8c>] cli_dlm_cncb+0x4dc/0x720 [parastor] [<ffffffff810ea9b9>] ? do_gettimeofday+0x29/0x70 [<ffffffffc1a257b0>] ? cli_dlm_need_cncb_func+0x60/0x60 [parastor] [<ffffffffc095c32e>] dlm_initr_call_cncb_do+0x13e/0x1c0 [dlm] [<ffffffffc0d7092d>] ? np_nrq_state_set.part.3+0x2d/0x80 [knal] [<ffffffffc0d74420>] ? np_nrq_notifystop+0x70/0x70 [knal] [<ffffffffc0d756e0>] ? np_nrq_nalctrl_put+0x100/0x100 [knal] [<ffffffffc0d755e0>] ? np_protoxtask_final+0x100/0x100 [knal] [<ffffffffc095dd88>] dlm_initr_call_cncb+0x2e8/0x470 [dlm] [<ffffffff810cd794>] ? update_curr+0x104/0x190 [<ffffffff810ca29e>] ? account_entity_dequeue+0xae/0xd0 [<ffffffff810cdc7c>] ? dequeue_entity+0x11c/0x5d0 [<ffffffffc0d75600>] ? np_nrq_nalctrl_put+0x20/0x100 [knal] [<ffffffffc0956f15>] dlm_initr_call_blcb+0x2d5/0x350 [dlm] [<ffffffff810ce54e>] ? dequeue_task_fair+0x41e/0x660 [<ffffffffc094621b>] ? dlm_lock_status_get+0x1b/0xd0 [dlm] [<ffffffff81029557>] ? __switch_to+0xd7/0x510 [<ffffffffc0957aab>] dlm_initr_callback+0xcb/0x2b0 [dlm] [<ffffffffc094614b>] ? dlm_lock_status_set+0x1b/0xd0 [dlm] [<ffffffffc094621b>] ? dlm_lock_status_get+0x1b/0xd0 [dlm] [<ffffffffc093db97>] ? dlm_memc_alloc+0x67/0x100 [dlm] [<ffffffffc095a345>] ? dlm_initr_lock_lru_del+0xa5/0xf0 [dlm] [<ffffffffc0958ec2>] dlm_initr_handle_cb+0xd22/0x1330 [dlm] [<ffffffff810b16d6>] ? finish_wait+0x56/0x70 [<ffffffff810b1910>] ? wake_up_atomic_t+0x30/0x30 [<ffffffffc0d74690>] ? np_nrq_timeout_notify+0x270/0x270 [knal] [<ffffffffc0d75ab0>] ? nwatch_command_cmpkey+0x60/0x60 [knal] [<ffffffffc0d75b20>] ? np_module_nalctrl_final+0x30/0x30 [knal] [<ffffffffc0950485>] dlm_initr_cb_tcb+0x135/0x450 [dlm] [<ffffffff810cdc7c>] ? dequeue_entity+0x11c/0x5d0 [<ffffffff811567bb>] ? probe_sched_wakeup+0x2b/0xa0 [<ffffffff810ce54e>] ? dequeue_task_fair+0x41e/0x660 [<ffffffff810c76f5>] ? sched_clock_cpu+0x85/0xc0 [<ffffffffc0943126>] dlm_tpool_tcb+0x16/0x20 [dlm] [<ffffffffc0e17347>] ofs_daemon_proxy+0x67/0x150 [knal] [<ffffffff816a8f8d>] ? __schedule+0x39d/0x8b0 [<ffffffffc0e172e0>] ? ofs_thread_daemonize+0x60/0x60 [knal] [<ffffffff810b098f>] kthread+0xcf/0xe0 [<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40 [<ffffffff816b4f18>] ret_from_fork+0x58/0x90 [<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40 Code: c8 c7 45 c4 00 00 00 00 48 63 80 a4 04 00 00 48 89 83 e0 00 00 00 4d 8b 7d 20 eb 2c 0f 1f 84 00 00 00 00 00 4d 8d 67 f0 49 8b 1f <4c> 89 e7 e8 21 9e 00 00 41 8b 47 68 85 c0 74 29 49 8d 47 10 49
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [dlm_i_cb/0:9900] Modules linked in: tun can_bcm can_raw can squashfs loop overlay(T) sctp_diag sctp dccp_diag dccp tcp_diag udp_diag inet_diag unix_diag ses enclosure fuse vfat fat isofs xfs uas usb_storage ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack libcrc32c iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter rctld(OE) parastor(OE) dlm(OE) kfnccli(OE) knal(OE) mmfs26(OE) mmfslinux(OE) tracedev(OE) rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_uverbs(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) iTCO_wdt iTCO_vendor_support skx_edac edac_core coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass joydev sg pcspkr shpchp i2c_i801 lpc_ich ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_cpufreq mei_me mei binfmt_misc nfsd knem(OE) auth_rpcgss nfs_acl lockd grace ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx5_ib(OE) ib_core(OE) 8021q garp stp crct10dif_pclmul llc crct10dif_common mrp crc32_pclmul crc32c_intel ghash_clmulni_intel i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt aesni_intel fb_sys_fops ttm lrw gf128mul glue_helper scsi_transport_iscsi ablk_helper drm cryptd ixgbe mpt3sas i40e raid_class ahci libahci dca mdio mlx5_core(OE) libata mlxfw(OE) mlx_compat(OE) devlink scsi_transport_sas ptp pps_core i2c_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod CPU: 2 PID: 9900 Comm: dlm_i_cb/0 Tainted: G W OEL ------------ T 3.10.0-693.el7.x86_64 #1 Hardware name: Sugon I620-G30/60P24-US, BIOS 0JGST031 08/02/2018 task: ffff880aeef60fd0 ti: ffff880af6750000 task.ti: ffff880af6750000 RIP: 0010:[<ffffffffc1a56b81>] [<ffffffffc1a56b81>] fsc_dentry_lru_del_init+0x31/0x100 [parastor] RSP: 0018:ffff880af6753780 EFLAGS: 00000286 RAX: ffff8816937b01b8 RBX: ffff880554668a00 RCX: dead000000000200 RDX: ffff880d80eb0400 RSI: ffff8816937b0290 RDI: ffff880d80eb0300 RBP: ffff880af67537d0 R08: ffff880d80eb0400 R09: 0000000000000000 R10: ffff880554668a00 R11: ffff88051ba96610 R12: ffff88095a93b980 R13: ffff880af6753770 R14: ffff880af67536f8 R15: 0000000000000202 FS: 0000000000000000(0000) GS:ffff880c0f680000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 00007f07c76f7000 CR3: 00000000019f2000 CR4: 00000000003407e0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 Stack: ffff880594078c00 ffff880594078c20 dead000000000200 ffff880594078c00 ffff8816937b0290 ffff880594078c00 ffffffffffffff10 ffffffffc1a4cd27 0000000000000010 ffff880d80eb0210 ffff880af6753850 ffffffffc1a4cd2f Call Trace: [<ffffffffc1a4cd27>] ? fsc_shrink_dcache_parent+0xb7/0x180 [parastor] [<ffffffffc1a4cd2f>] fsc_shrink_dcache_parent+0xbf/0x180 [parastor] [<ffffffffc1a57f5c>] fscex_drop_negative_dentries+0x86c/0x910 [parastor] [<ffffffffc1b3214a>] ? fsc_storsys_find_bykey+0x3a/0xc0 [parastor] [<ffffffffc1b0b595>] ? cli_finode_xattr_reclaim+0x225/0x580 [parastor] [<ffffffffc1acdc8c>] cli_dlm_cncb+0x4dc/0x720 [parastor] [<ffffffffc0e2152a>] ? np_protoxtask_new+0x3a/0x370 [knal] [<ffffffffc1acd7b0>] ? cli_dlm_need_cncb_func+0x60/0x60 [parastor] [<ffffffffc0a1632e>] dlm_initr_call_cncb_do+0x13e/0x1c0 [dlm] [<ffffffffc0e1892d>] ? np_nrq_state_set.part.3+0x2d/0x80 [knal] [<ffffffffc0e1c420>] ? np_nrq_notifystop+0x70/0x70 [knal] [<ffffffffc0e1d6e0>] ? np_nrq_nalctrl_put+0x100/0x100 [knal] [<ffffffffc0e1d5e0>] ? np_protoxtask_final+0x100/0x100 [knal] [<ffffffffc0a17d88>] dlm_initr_call_cncb+0x2e8/0x470 [dlm] [<ffffffff810cd794>] ? update_curr+0x104/0x190 [<ffffffff810ca29e>] ? account_entity_dequeue+0xae/0xd0 [<ffffffff810cdc7c>] ? dequeue_entity+0x11c/0x5d0 [<ffffffffc0e1d600>] ? np_nrq_nalctrl_put+0x20/0x100 [knal] [<ffffffffc0a10f15>] dlm_initr_call_blcb+0x2d5/0x350 [dlm] [<ffffffff810ce54e>] ? dequeue_task_fair+0x41e/0x660 [<ffffffffc0a0021b>] ? dlm_lock_status_get+0x1b/0xd0 [dlm] [<ffffffff81029557>] ? __switch_to+0xd7/0x510 [<ffffffffc0a11aab>] dlm_initr_callback+0xcb/0x2b0 [dlm] [<ffffffffc0a0014b>] ? dlm_lock_status_set+0x1b/0xd0 [dlm] [<ffffffffc0a0021b>] ? dlm_lock_status_get+0x1b/0xd0 [dlm] [<ffffffffc09f7b97>] ? dlm_memc_alloc+0x67/0x100 [dlm] [<ffffffffc0a14345>] ? dlm_initr_lock_lru_del+0xa5/0xf0 [dlm] [<ffffffffc0a12ec2>] dlm_initr_handle_cb+0xd22/0x1330 [dlm] [<ffffffff810b16d6>] ? finish_wait+0x56/0x70 [<ffffffff810b1910>] ? wake_up_atomic_t+0x30/0x30 [<ffffffffc0e1c690>] ? np_nrq_timeout_notify+0x270/0x270 [knal] [<ffffffffc0e1dab0>] ? nwatch_command_cmpkey+0x60/0x60 [knal] [<ffffffffc0e1db20>] ? np_module_nalctrl_final+0x30/0x30 [knal] [<ffffffffc0a0a485>] dlm_initr_cb_tcb+0x135/0x450 [dlm] [<ffffffff810cdc7c>] ? dequeue_entity+0x11c/0x5d0 [<ffffffff810ce54e>] ? dequeue_task_fair+0x41e/0x660 [<ffffffff810c76f5>] ? sched_clock_cpu+0x85/0xc0 [<ffffffffc09fd126>] dlm_tpool_tcb+0x16/0x20 [dlm] [<ffffffffc0ebf347>] ofs_daemon_proxy+0x67/0x150 [knal] [<ffffffff816a8f8d>] ? __schedule+0x39d/0x8b0 [<ffffffffc0ebf2e0>] ? ofs_thread_daemonize+0x60/0x60 [knal] [<ffffffff810b098f>] kthread+0xcf/0xe0 [<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40 [<ffffffff816b4f18>] ret_from_fork+0x58/0x90 [<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40 Code: 48 89 e5 53 48 89 fb 48 83 ec 48 48 85 ff 0f 84 d6 00 00 00 48 8b 87 98 00 00 00 48 85 c0 0f 84 c4 00 00 00 48 8b 80 f8 00 00 00 <48> 85 c0 0f 84 b2 00 00 00 48 3b 3f 0f 84 91 00 00 00 83 a8 d4
NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [dlm_i_cb/3:3410] Modules linked in: ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables squashfs loop overlay(T) can_bcm sctp can_raw can iptable_filter mmfs26(OE) mmfslinux(OE) tracedev(OE) vfat fat isofs xfs libcrc32c uas usb_storage tcp_lp fuse rctld(OE) parastor(OE) dlm(OE) kfnccli(OE) knal(OE) rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_uverbs(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) iTCO_wdt iTCO_vendor_support skx_edac edac_core coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass pcspkr joydev sg i2c_i801 lpc_ich mei_me shpchp mei ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_cpufreq nfsd auth_rpcgss nfs_acl knem(OE) lockd binfmt_misc grace ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx5_ib(OE) ib_core(OE) 8021q garp stp llc mrp crct10dif_pclmul crct10dif_common crc32_pclmul crc32c_intel scsi_transport_iscsi ghash_clmulni_intel i2c_algo_bit drm_kms_helper ahci libahci ixgbe mpt3sas syscopyarea sysfillrect sysimgblt fb_sys_fops aesni_intel ttm lrw gf128mul glue_helper ablk_helper drm cryptd i40e dca mdio raid_class libata mlx5_core(OE) scsi_transport_sas mlxfw(OE) mlx_compat(OE) devlink ptp pps_core i2c_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod [last unloaded: tracedev] CPU: 5 PID: 3410 Comm: dlm_i_cb/3 Tainted: G W OE ------------ T 3.10.0-693.el7.x86_64 #1 Hardware name: Sugon I620-G30/60P24-US, BIOS 0JGST031 08/02/2018 task: ffff8817f5f68fd0 ti: ffff8817404e8000 task.ti: ffff8817404e8000 RIP: 0010:[<ffffffff81331a89>] [<ffffffff81331a89>] __write_lock_failed+0x9/0x20 RSP: 0018:ffff8817404eb570 EFLAGS: 00000297 RAX: ffff88173bb44000 RBX: 0000000000000010 RCX: dead000000000200 RDX: 0000000000000001 RSI: ffff88115f84a890 RDI: ffff880bb9f74294 RBP: ffff8817404eb570 R08: ffff88180a450450 R09: 000000018040001b R10: ffff88180a4503c0 R11: ffffea0059542900 R12: ffff8817404eb550 R13: ffff8816550a6400 R14: ffff8816550a6410 R15: ffffffffc1710be7 FS: 0000000000000000(0000) GS:ffff88180da40000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 00005559e46e0000 CR3: 00000003aa936000 CR4: 00000000003407e0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 Stack: ffff8817404eb580 ffffffff816ab927 ffff8817404eb5f0 ffffffffc1711147 ffffffffffffff10 ffff880bb9f741b8 0000000000000010 0000000000000282 ffff8817404eb5c8 000000002d595303 ffff8817404eb700 ffff88180a450d80 Call Trace: [<ffffffff816ab927>] _raw_write_lock+0x17/0x20 [<ffffffffc1711147>] fsc_d_drop+0x57/0x160 [parastor] [<ffffffffc1705bc4>] lcli_d_drop_fde+0xd4/0x190 [parastor] [<ffffffffc1705e0e>] lcli_d_release+0xbe/0xe0 [parastor] [<ffffffff8121ddcc>] ? inode_add_lru+0x3c/0xa0 [<ffffffff8121df73>] ? iput+0x143/0x190 [<ffffffff81218188>] d_free+0x38/0x70 [<ffffffff8121a35f>] shrink_dentry_list+0x21f/0x490 [<ffffffff8121a6e8>] shrink_dcache_parent+0x38/0x90 [<ffffffffc1706f36>] lcli_drop_negvde+0x616/0x650 [parastor] [<ffffffffc17398ed>] ? blkgrp_vec_lookup+0x1d/0x30 [parastor] [<ffffffffc171ee5d>] ? truncate_inode_blkgrps_range_wait+0xcd/0x270 [parastor] [<ffffffffc17129fe>] fscex_drop_negative_dentries+0x30e/0x910 [parastor] [<ffffffffc171f495>] ? __truncate_inode_blks_range+0xc5/0xe0 [parastor] [<ffffffffc17879ab>] ? cli_dlm_cncb_drh2finode+0x6b/0x180 [parastor] [<ffffffffc1788c8c>] cli_dlm_cncb+0x4dc/0x720 [parastor] [<ffffffff810ea9b9>] ? do_gettimeofday+0x29/0x70 [<ffffffffc17887b0>] ? cli_dlm_need_cncb_func+0x60/0x60 [parastor] [<ffffffffc08fe32e>] dlm_initr_call_cncb_do+0x13e/0x1c0 [dlm] [<ffffffffc0ad392d>] ? np_nrq_state_set.part.3+0x2d/0x80 [knal] [<ffffffffc0ad7420>] ? np_nrq_notifystop+0x70/0x70 [knal] [<ffffffffc0ad86e0>] ? np_nrq_nalctrl_put+0x100/0x100 [knal] [<ffffffffc0ad85e0>] ? np_protoxtask_final+0x100/0x100 [knal] [<ffffffffc08ffd88>] dlm_initr_call_cncb+0x2e8/0x470 [dlm] [<ffffffff810cd794>] ? update_curr+0x104/0x190 [<ffffffff810ca29e>] ? account_entity_dequeue+0xae/0xd0 [<ffffffff810cdc7c>] ? dequeue_entity+0x11c/0x5d0 [<ffffffffc0ad8600>] ? np_nrq_nalctrl_put+0x20/0x100 [knal] [<ffffffffc08f8f15>] dlm_initr_call_blcb+0x2d5/0x350 [dlm] [<ffffffff810ce54e>] ? dequeue_task_fair+0x41e/0x660 [<ffffffffc08e821b>] ? dlm_lock_status_get+0x1b/0xd0 [dlm] [<ffffffff81029557>] ? __switch_to+0xd7/0x510 [<ffffffffc08f9aab>] dlm_initr_callback+0xcb/0x2b0 [dlm] [<ffffffffc08e814b>] ? dlm_lock_status_set+0x1b/0xd0 [dlm] [<ffffffffc08e821b>] ? dlm_lock_status_get+0x1b/0xd0 [dlm] [<ffffffffc08dfb97>] ? dlm_memc_alloc+0x67/0x100 [dlm] [<ffffffffc08fc345>] ? dlm_initr_lock_lru_del+0xa5/0xf0 [dlm] [<ffffffffc08faec2>] dlm_initr_handle_cb+0xd22/0x1330 [dlm] [<ffffffff810b16d6>] ? finish_wait+0x56/0x70 [<ffffffff810b1910>] ? wake_up_atomic_t+0x30/0x30 [<ffffffffc0ad7690>] ? np_nrq_timeout_notify+0x270/0x270 [knal] [<ffffffffc0ad8ab0>] ? nwatch_command_cmpkey+0x60/0x60 [knal] [<ffffffffc0ad8b20>] ? np_module_nalctrl_final+0x30/0x30 [knal] [<ffffffffc08f2485>] dlm_initr_cb_tcb+0x135/0x450 [dlm] [<ffffffff810cdc7c>] ? dequeue_entity+0x11c/0x5d0 [<ffffffff810ce54e>] ? dequeue_task_fair+0x41e/0x660 [<ffffffff810c76f5>] ? sched_clock_cpu+0x85/0xc0 [<ffffffffc08e5126>] dlm_tpool_tcb+0x16/0x20 [dlm] [<ffffffffc0b7a347>] ofs_daemon_proxy+0x67/0x150 [knal] [<ffffffff816a8f8d>] ? __schedule+0x39d/0x8b0 [<ffffffffc0b7a2e0>] ? ofs_thread_daemonize+0x60/0x60 [knal] [<ffffffff810b098f>] kthread+0xcf/0xe0 [<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40 [<ffffffff816b4f18>] ret_from_fork+0x58/0x90 [<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40 Code: 01 cb 48 89 01 31 c0 0f 01 ca c3 b8 f2 ff ff ff 0f 01 ca c3 90 90 90 90 90 90 90 90 90 90 90 90 90 90 55 48 89 e5 f0 ff 07 f3 90 <83> 3f 01 75 f9 f0 ff 0f 75 f1 5d c3 90 66 2e 0f 1f 84 00 00 00
NMI watchdog: BUG: soft lockup - CPU#8 stuck for 22s! [dlm_i_cb/0:5164] Modules linked in: nfsv3 rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache ps200b(OE) parastor(OE) dlm(OE) kfnccli(OE) knal(OE) nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter overlay(T) rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_uverbs(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx5_ib(OE) mlx5_core(OE) mlxfw(OE) mlx4_en(OE) dm_mirror dm_region_hash dm_log dm_mod sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul iTCO_wdt joydev glue_helper ablk_helper ipmi_si iTCO_vendor_support cryptd ipmi_devintf ipmi_msghandler pcspkr sg shpchp i2c_i801 mei_me mei wmi lpc_ich nfsd auth_rpcgss nfs_acl lockd grace sunrpc knem(OE) ip_tables ext4 mbcache jbd2 mlx4_ib(OE) ib_core(OE) sd_mod sr_mod cdrom crc_t10dif crct10dif_generic ast drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm drm libsas ahci igb crct10dif_pclmul e1000e scsi_transport_sas libahci crct10dif_common crc32c_intel libata mlx4_core(OE) devlink megaraid_sas mlx_compat(OE) dca ptp i2c_algo_bit pps_core i2c_core CPU: 8 PID: 5164 Comm: dlm_i_cb/0 Tainted: G OEL ------------ T 3.10.0-693.el7.x86_64 #1 Hardware name: empty empty/S7056 , BIOS V7.123 12/02/2013 task: ffff8808323f0fd0 ti: ffff88077d548000 task.ti: ffff88077d548000 RIP: 0010:[<ffffffffc179dd37>] [<ffffffffc179dd37>] fsc_shrink_dcache_parent+0xb7/0x190 [parastor] RSP: 0018:ffff88077d54b7e0 EFLAGS: 00000287 RAX: ffff8801ba2a0320 RBX: 0000000000000010 RCX: dead000000000200 RDX: ffff8801ba2a0300 RSI: ffff88085dbac290 RDI: ffff8801ba2a0300 RBP: ffff88077d54b850 R08: ffff8801ba2a0300 R09: ffff88045df48880 R10: ffff8805cdf11d00 R11: ffff8805b15bcc10 R12: ffffffffffffff10 R13: ffff8807d1d61000 R14: ffff88085dbac290 R15: ffff8807d1d61000 FS: 0000000000000000(0000) GS:ffff88087f680000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 00007f7474c4e000 CR3: 00000000019f2000 CR4: 00000000001407e0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 Stack: ffff88048276f100 ffff8803da384000 ffff8808323f0fd0 ffff8805cdf11d20 ffff88085dbac290 ffff88085dbac1b8 000000004341c000 ffff8805cdf11d00 ffff88085dbac290 000000000001b463 0000000000001321 ffff88024341bfc0 Call Trace: [<ffffffffc17a935c>] fscex_drop_negative_dentries+0x86c/0x920 [parastor] [<ffffffffc188564a>] ? fsc_storsys_find_bykey+0x3a/0xc0 [parastor] [<ffffffffc185e605>] ? cli_finode_xattr_reclaim+0x225/0x590 [parastor] [<ffffffffc182092c>] cli_dlm_cncb+0x4dc/0x740 [parastor] [<ffffffff810ea9b9>] ? do_gettimeofday+0x29/0x70 [<ffffffffc1820450>] ? cli_dlm_need_cncb_func+0x60/0x60 [parastor] [<ffffffffc157b51e>] dlm_initr_call_cncb_do+0x13e/0x1c0 [dlm] [<ffffffffc095ed1d>] ? np_nrq_state_set.part.3+0x2d/0x80 [knal] [<ffffffffc0962880>] ? np_nrq_notifystop+0x70/0x70 [knal] [<ffffffffc0963b60>] ? np_nrq_nalctrl_put+0x100/0x100 [knal] [<ffffffffc0963a60>] ? np_protoxtask_final+0x100/0x100 [knal] [<ffffffffc157cf78>] dlm_initr_call_cncb+0x2e8/0x470 [dlm] [<ffffffff810cd794>] ? update_curr+0x104/0x190 [<ffffffff810ca29e>] ? account_entity_dequeue+0xae/0xd0 [<ffffffff810cdc7c>] ? dequeue_entity+0x11c/0x5d0 [<ffffffffc15760b5>] dlm_initr_call_blcb+0x2d5/0x350 [dlm] [<ffffffff810ce54e>] ? dequeue_task_fair+0x41e/0x660 [<ffffffffc15651cb>] ? dlm_lock_status_get+0x1b/0xd0 [dlm] [<ffffffff81029557>] ? __switch_to+0xd7/0x510 [<ffffffffc1576c5b>] dlm_initr_callback+0xcb/0x2b0 [dlm] [<ffffffffc15650fb>] ? dlm_lock_status_set+0x1b/0xd0 [dlm] [<ffffffffc15651cb>] ? dlm_lock_status_get+0x1b/0xd0 [dlm] [<ffffffffc155cb9f>] ? dlm_memc_alloc+0x6f/0x100 [dlm] [<ffffffffc1579525>] ? dlm_initr_lock_lru_del+0xa5/0xf0 [dlm] [<ffffffffc1578072>] dlm_initr_handle_cb+0xd22/0x1350 [dlm] [<ffffffff810b16d6>] ? finish_wait+0x56/0x70 [<ffffffff810b1910>] ? wake_up_atomic_t+0x30/0x30 [<ffffffffc0962af0>] ? np_nrq_timeout_notify+0x270/0x270 [knal] [<ffffffffc0963f50>] ? nwatch_command_cmpkey+0x60/0x60 [knal] [<ffffffffc0963fd0>] ? np_module_nalctrl_final+0x30/0x30 [knal] [<ffffffffc156f505>] dlm_initr_cb_tcb+0x135/0x480 [dlm] [<ffffffff810cdc7c>] ? dequeue_entity+0x11c/0x5d0 [<ffffffff810ce54e>] ? dequeue_task_fair+0x41e/0x660 [<ffffffff810c76f5>] ? sched_clock_cpu+0x85/0xc0 [<ffffffffc1562126>] dlm_tpool_tcb+0x16/0x20 [dlm] [<ffffffffc0a06367>] ofs_daemon_proxy+0x67/0x150 [knal] [<ffffffff816a8f8d>] ? __schedule+0x39d/0x8b0 [<ffffffffc0a06300>] ? ofs_thread_daemonize+0x60/0x60 [knal] [<ffffffff810b098f>] kthread+0xcf/0xe0 [<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40 [<ffffffff816b4f18>] ret_from_fork+0x58/0x90 [<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40 Code: c8 c7 45 c4 00 00 00 00 48 63 80 a4 04 00 00 48 89 83 e0 00 00 00 4d 8b 7d 20 eb 2c 0f 1f 84 00 00 00 00 00 4d 8d 67 f0 49 8b 1f <4c> 89 e7 e8 b1 a1 00 00 41 8b 47 68 85 c0 74 29 49 8d 47 10 49