Info

Permalink
Function
fsc_shrink_dcache_parent
First occurrence
2021-04-26
Last occurrence
2026-02-27
State
NEW
Components
kernel Tainted
Quality
-117

Statistics

Operating system Count
CentOS 7 10
Architecture Count
x86_64 10
Related packages Count
kernel 10
 0:3.10.0-693.el7 10

Packages names with count less than the total count of reports are most probably not the cause of the problem.

History

Daily:

Weekly:

Monthly:

Report backtrace


Complete report #1401415
Frame # Function Binary Source or offset Line
1
fsc_shrink_dcache_parent
parastor 0xffffffffc19a4d27
Build id: 3.10.0-693.el7.x86_64
0xb7
2
fscex_drop_negative_dentries
parastor 0xffffffffc19aff5c
Build id: 3.10.0-693.el7.x86_64
0x86c
3
? __truncate_inode_blks_range
parastor 0xffffffffc19bc495
Build id: 3.10.0-693.el7.x86_64
0xc5
4
cli_dlm_cncb
parastor 0xffffffffc1a25c8c
Build id: 3.10.0-693.el7.x86_64
0x4dc
5
? do_gettimeofday
vmlinux 0xffffffff810ea9b9
Build id: 3.10.0-693.el7.x86_64
0x29
6
? cli_dlm_need_cncb_func
parastor 0xffffffffc1a257b0
Build id: 3.10.0-693.el7.x86_64
0x60
7
dlm_initr_call_cncb_do
dlm 0xffffffffc095c32e
Build id: 3.10.0-693.el7.x86_64
0x13e
8
? np_nrq_state_set.part.3
knal 0xffffffffc0d7092d
Build id: 3.10.0-693.el7.x86_64
0x2d
9
? np_nrq_notifystop
knal 0xffffffffc0d74420
Build id: 3.10.0-693.el7.x86_64
0x70
10
? np_nrq_nalctrl_put
knal 0xffffffffc0d756e0
Build id: 3.10.0-693.el7.x86_64
0x100
11
? np_protoxtask_final
knal 0xffffffffc0d755e0
Build id: 3.10.0-693.el7.x86_64
0x100
12
dlm_initr_call_cncb
dlm 0xffffffffc095dd88
Build id: 3.10.0-693.el7.x86_64
0x2e8
13
? update_curr
vmlinux 0xffffffff810cd794
Build id: 3.10.0-693.el7.x86_64
0x104
14
? account_entity_dequeue
vmlinux 0xffffffff810ca29e
Build id: 3.10.0-693.el7.x86_64
0xae
15
? dequeue_entity
vmlinux 0xffffffff810cdc7c
Build id: 3.10.0-693.el7.x86_64
0x11c
16
? np_nrq_nalctrl_put
knal 0xffffffffc0d75600
Build id: 3.10.0-693.el7.x86_64
0x20
17
dlm_initr_call_blcb
dlm 0xffffffffc0956f15
Build id: 3.10.0-693.el7.x86_64
0x2d5
18
? dequeue_task_fair
vmlinux 0xffffffff810ce54e
Build id: 3.10.0-693.el7.x86_64
0x41e
19
? dlm_lock_status_get
dlm 0xffffffffc094621b
Build id: 3.10.0-693.el7.x86_64
0x1b
20
? __switch_to
vmlinux 0xffffffff81029557
Build id: 3.10.0-693.el7.x86_64
0xd7
21
dlm_initr_callback
dlm 0xffffffffc0957aab
Build id: 3.10.0-693.el7.x86_64
0xcb
22
? dlm_lock_status_set
dlm 0xffffffffc094614b
Build id: 3.10.0-693.el7.x86_64
0x1b
23
? dlm_lock_status_get
dlm 0xffffffffc094621b
Build id: 3.10.0-693.el7.x86_64
0x1b
24
? dlm_memc_alloc
dlm 0xffffffffc093db97
Build id: 3.10.0-693.el7.x86_64
0x67
25
? dlm_initr_lock_lru_del
dlm 0xffffffffc095a345
Build id: 3.10.0-693.el7.x86_64
0xa5
26
dlm_initr_handle_cb
dlm 0xffffffffc0958ec2
Build id: 3.10.0-693.el7.x86_64
0xd22
27
? finish_wait
vmlinux 0xffffffff810b16d6
Build id: 3.10.0-693.el7.x86_64
0x56
28
? wake_up_atomic_t
vmlinux 0xffffffff810b1910
Build id: 3.10.0-693.el7.x86_64
0x30
29
? np_nrq_timeout_notify
knal 0xffffffffc0d74690
Build id: 3.10.0-693.el7.x86_64
0x270
30
? nwatch_command_cmpkey
knal 0xffffffffc0d75ab0
Build id: 3.10.0-693.el7.x86_64
0x60
31
? np_module_nalctrl_final
knal 0xffffffffc0d75b20
Build id: 3.10.0-693.el7.x86_64
0x30
32
dlm_initr_cb_tcb
dlm 0xffffffffc0950485
Build id: 3.10.0-693.el7.x86_64
0x135
33
? dequeue_entity
vmlinux 0xffffffff810cdc7c
Build id: 3.10.0-693.el7.x86_64
0x11c
34
? probe_sched_wakeup
vmlinux 0xffffffff811567bb
Build id: 3.10.0-693.el7.x86_64
0x2b
35
? dequeue_task_fair
vmlinux 0xffffffff810ce54e
Build id: 3.10.0-693.el7.x86_64
0x41e
36
? sched_clock_cpu
vmlinux 0xffffffff810c76f5
Build id: 3.10.0-693.el7.x86_64
0x85
37
dlm_tpool_tcb
dlm 0xffffffffc0943126
Build id: 3.10.0-693.el7.x86_64
0x16
38
ofs_daemon_proxy
knal 0xffffffffc0e17347
Build id: 3.10.0-693.el7.x86_64
0x67
39
? __schedule
vmlinux 0xffffffff816a8f8d
Build id: 3.10.0-693.el7.x86_64
0x39d
40
? ofs_thread_daemonize
knal 0xffffffffc0e172e0
Build id: 3.10.0-693.el7.x86_64
0x60
41
kthread
vmlinux 0xffffffff810b098f
Build id: 3.10.0-693.el7.x86_64
0xcf
42
? insert_kthread_work
vmlinux 0xffffffff810b08c0
Build id: 3.10.0-693.el7.x86_64
0x40
43
ret_from_fork
vmlinux 0xffffffff816b4f18
Build id: 3.10.0-693.el7.x86_64
0x58
44
? insert_kthread_work
vmlinux 0xffffffff810b08c0
Build id: 3.10.0-693.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [dlm_i_cb/2:10024]
Modules linked in: nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter tcp_lp fuse squashfs loop overlay(T) rctld(OE) parastor(OE) dlm(OE) kfnccli(OE) knal(OE) mmfs26(OE) mmfslinux(OE) tracedev(OE) rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_uverbs(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) skx_edac edac_core coretemp iTCO_wdt intel_rapl iTCO_vendor_support iosf_mbi kvm_intel kvm irqbypass sg joydev pcspkr mei_me shpchp lpc_ich mei i2c_i801 ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_cpufreq binfmt_misc knem(OE) nfsd auth_rpcgss nfs_acl lockd grace ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx5_ib(OE) ib_core(OE) 8021q garp stp llc mrp crct10dif_pclmul crct10dif_common
crc32_pclmul ixgbe crc32c_intel ghash_clmulni_intel i2c_algo_bit drm_kms_helper aesni_intel lrw gf128mul glue_helper ablk_helper cryptd syscopyarea sysfillrect sysimgblt fb_sys_fops ttm dca drm mdio ahci libahci i40e scsi_transport_iscsi libata mpt3sas mlx5_core(OE) i2c_core mlxfw(OE) mlx_compat(OE) devlink ptp raid_class pps_core scsi_transport_sas wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 10024 Comm: dlm_i_cb/2 Tainted: G        W  OEL ------------ T 3.10.0-693.el7.x86_64 #1
Hardware name: Sugon I620-G30/60P24-US, BIOS 0JGST031 08/02/2018
task: ffff8816f16b9fa0 ti: ffff8816410a8000 task.ti: ffff8816410a8000
RIP: 0010:[<ffffffffc19a4d27>]  [<ffffffffc19a4d27>] fsc_shrink_dcache_parent+0xb7/0x180 [parastor]
RSP: 0018:ffff8816410ab7e0  EFLAGS: 00000287
RAX: ffff880f0be99820 RBX: ffff881000472500 RCX: dead000000000200
RDX: ffff880f0be99800 RSI: ffff8816e172c290 RDI: ffff880f0be99800
RBP: ffff8816410ab850 R08: ffff880f0be99800 R09: 0000000000000002
R10: ffff881000472500 R11: ffff881028c42310 R12: ffff881000472500
R13: ffff881028c42310 R14: ffff8816e172c290 R15: ffff8816410ab7c0
FS:  0000000000000000(0000) GS:ffff880c0f600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000000020a8008 CR3: 00000000019f2000 CR4: 00000000003407f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffff880e11bd6500 ffff881626590000 ffff8816f16b9fa0 ffff881000472520
ffff8816e172c290 ffff8816e172c1b8 00000000ffffff10 ffff881000472500
0000000000000010 000000000000d580 0000000000000d3a ffff880fe6d21540
Call Trace:
[<ffffffffc19aff5c>] fscex_drop_negative_dentries+0x86c/0x910 [parastor]
[<ffffffffc19bc495>] ? __truncate_inode_blks_range+0xc5/0xe0 [parastor]
[<ffffffffc1a25c8c>] cli_dlm_cncb+0x4dc/0x720 [parastor]
[<ffffffff810ea9b9>] ? do_gettimeofday+0x29/0x70
[<ffffffffc1a257b0>] ? cli_dlm_need_cncb_func+0x60/0x60 [parastor]
[<ffffffffc095c32e>] dlm_initr_call_cncb_do+0x13e/0x1c0 [dlm]
[<ffffffffc0d7092d>] ? np_nrq_state_set.part.3+0x2d/0x80 [knal]
[<ffffffffc0d74420>] ? np_nrq_notifystop+0x70/0x70 [knal]
[<ffffffffc0d756e0>] ? np_nrq_nalctrl_put+0x100/0x100 [knal]
[<ffffffffc0d755e0>] ? np_protoxtask_final+0x100/0x100 [knal]
[<ffffffffc095dd88>] dlm_initr_call_cncb+0x2e8/0x470 [dlm]
[<ffffffff810cd794>] ? update_curr+0x104/0x190
[<ffffffff810ca29e>] ? account_entity_dequeue+0xae/0xd0
[<ffffffff810cdc7c>] ? dequeue_entity+0x11c/0x5d0
[<ffffffffc0d75600>] ? np_nrq_nalctrl_put+0x20/0x100 [knal]
[<ffffffffc0956f15>] dlm_initr_call_blcb+0x2d5/0x350 [dlm]
[<ffffffff810ce54e>] ? dequeue_task_fair+0x41e/0x660
[<ffffffffc094621b>] ? dlm_lock_status_get+0x1b/0xd0 [dlm]
[<ffffffff81029557>] ? __switch_to+0xd7/0x510
[<ffffffffc0957aab>] dlm_initr_callback+0xcb/0x2b0 [dlm]
[<ffffffffc094614b>] ? dlm_lock_status_set+0x1b/0xd0 [dlm]
[<ffffffffc094621b>] ? dlm_lock_status_get+0x1b/0xd0 [dlm]
[<ffffffffc093db97>] ? dlm_memc_alloc+0x67/0x100 [dlm]
[<ffffffffc095a345>] ? dlm_initr_lock_lru_del+0xa5/0xf0 [dlm]
[<ffffffffc0958ec2>] dlm_initr_handle_cb+0xd22/0x1330 [dlm]
[<ffffffff810b16d6>] ? finish_wait+0x56/0x70
[<ffffffff810b1910>] ? wake_up_atomic_t+0x30/0x30
[<ffffffffc0d74690>] ? np_nrq_timeout_notify+0x270/0x270 [knal]
[<ffffffffc0d75ab0>] ? nwatch_command_cmpkey+0x60/0x60 [knal]
[<ffffffffc0d75b20>] ? np_module_nalctrl_final+0x30/0x30 [knal]
[<ffffffffc0950485>] dlm_initr_cb_tcb+0x135/0x450 [dlm]
[<ffffffff810cdc7c>] ? dequeue_entity+0x11c/0x5d0
[<ffffffff811567bb>] ? probe_sched_wakeup+0x2b/0xa0
[<ffffffff810ce54e>] ? dequeue_task_fair+0x41e/0x660
[<ffffffff810c76f5>] ? sched_clock_cpu+0x85/0xc0
[<ffffffffc0943126>] dlm_tpool_tcb+0x16/0x20 [dlm]
[<ffffffffc0e17347>] ofs_daemon_proxy+0x67/0x150 [knal]
[<ffffffff816a8f8d>] ? __schedule+0x39d/0x8b0
[<ffffffffc0e172e0>] ? ofs_thread_daemonize+0x60/0x60 [knal]
[<ffffffff810b098f>] kthread+0xcf/0xe0
[<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
[<ffffffff816b4f18>] ret_from_fork+0x58/0x90
[<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
Code: c8 c7 45 c4 00 00 00 00 48 63 80 a4 04 00 00 48 89 83 e0 00 00 00 4d 8b 7d 20 eb 2c 0f 1f 84 00 00 00 00 00 4d 8d 67 f0 49 8b 1f <4c> 89 e7 e8 21 9e 00 00 41 8b 47 68 85 c0 74 29 49 8d 47 10 49 
Complete report #913372
Frame # Function Binary Source or offset Line
1
fsc_dentry_lru_del_init
parastor 0xffffffffc1a56b81
Build id: 3.10.0-693.el7.x86_64
0x31
2
? fsc_shrink_dcache_parent
parastor 0xffffffffc1a4cd27
Build id: 3.10.0-693.el7.x86_64
0xb7
3
fsc_shrink_dcache_parent
parastor 0xffffffffc1a4cd2f
Build id: 3.10.0-693.el7.x86_64
0xbf
4
fscex_drop_negative_dentries
parastor 0xffffffffc1a57f5c
Build id: 3.10.0-693.el7.x86_64
0x86c
5
? fsc_storsys_find_bykey
parastor 0xffffffffc1b3214a
Build id: 3.10.0-693.el7.x86_64
0x3a
6
? cli_finode_xattr_reclaim
parastor 0xffffffffc1b0b595
Build id: 3.10.0-693.el7.x86_64
0x225
7
cli_dlm_cncb
parastor 0xffffffffc1acdc8c
Build id: 3.10.0-693.el7.x86_64
0x4dc
8
? np_protoxtask_new
knal 0xffffffffc0e2152a
Build id: 3.10.0-693.el7.x86_64
0x3a
9
? cli_dlm_need_cncb_func
parastor 0xffffffffc1acd7b0
Build id: 3.10.0-693.el7.x86_64
0x60
10
dlm_initr_call_cncb_do
dlm 0xffffffffc0a1632e
Build id: 3.10.0-693.el7.x86_64
0x13e
11
? np_nrq_state_set.part.3
knal 0xffffffffc0e1892d
Build id: 3.10.0-693.el7.x86_64
0x2d
12
? np_nrq_notifystop
knal 0xffffffffc0e1c420
Build id: 3.10.0-693.el7.x86_64
0x70
13
? np_nrq_nalctrl_put
knal 0xffffffffc0e1d6e0
Build id: 3.10.0-693.el7.x86_64
0x100
14
? np_protoxtask_final
knal 0xffffffffc0e1d5e0
Build id: 3.10.0-693.el7.x86_64
0x100
15
dlm_initr_call_cncb
dlm 0xffffffffc0a17d88
Build id: 3.10.0-693.el7.x86_64
0x2e8
16
? update_curr
vmlinux 0xffffffff810cd794
Build id: 3.10.0-693.el7.x86_64
0x104
17
? account_entity_dequeue
vmlinux 0xffffffff810ca29e
Build id: 3.10.0-693.el7.x86_64
0xae
18
? dequeue_entity
vmlinux 0xffffffff810cdc7c
Build id: 3.10.0-693.el7.x86_64
0x11c
19
? np_nrq_nalctrl_put
knal 0xffffffffc0e1d600
Build id: 3.10.0-693.el7.x86_64
0x20
20
dlm_initr_call_blcb
dlm 0xffffffffc0a10f15
Build id: 3.10.0-693.el7.x86_64
0x2d5
21
? dequeue_task_fair
vmlinux 0xffffffff810ce54e
Build id: 3.10.0-693.el7.x86_64
0x41e
22
? dlm_lock_status_get
dlm 0xffffffffc0a0021b
Build id: 3.10.0-693.el7.x86_64
0x1b
23
? __switch_to
vmlinux 0xffffffff81029557
Build id: 3.10.0-693.el7.x86_64
0xd7
24
dlm_initr_callback
dlm 0xffffffffc0a11aab
Build id: 3.10.0-693.el7.x86_64
0xcb
25
? dlm_lock_status_set
dlm 0xffffffffc0a0014b
Build id: 3.10.0-693.el7.x86_64
0x1b
26
? dlm_lock_status_get
dlm 0xffffffffc0a0021b
Build id: 3.10.0-693.el7.x86_64
0x1b
27
? dlm_memc_alloc
dlm 0xffffffffc09f7b97
Build id: 3.10.0-693.el7.x86_64
0x67
28
? dlm_initr_lock_lru_del
dlm 0xffffffffc0a14345
Build id: 3.10.0-693.el7.x86_64
0xa5
29
dlm_initr_handle_cb
dlm 0xffffffffc0a12ec2
Build id: 3.10.0-693.el7.x86_64
0xd22
30
? finish_wait
vmlinux 0xffffffff810b16d6
Build id: 3.10.0-693.el7.x86_64
0x56
31
? wake_up_atomic_t
vmlinux 0xffffffff810b1910
Build id: 3.10.0-693.el7.x86_64
0x30
32
? np_nrq_timeout_notify
knal 0xffffffffc0e1c690
Build id: 3.10.0-693.el7.x86_64
0x270
33
? nwatch_command_cmpkey
knal 0xffffffffc0e1dab0
Build id: 3.10.0-693.el7.x86_64
0x60
34
? np_module_nalctrl_final
knal 0xffffffffc0e1db20
Build id: 3.10.0-693.el7.x86_64
0x30
35
dlm_initr_cb_tcb
dlm 0xffffffffc0a0a485
Build id: 3.10.0-693.el7.x86_64
0x135
36
? dequeue_entity
vmlinux 0xffffffff810cdc7c
Build id: 3.10.0-693.el7.x86_64
0x11c
37
? dequeue_task_fair
vmlinux 0xffffffff810ce54e
Build id: 3.10.0-693.el7.x86_64
0x41e
38
? sched_clock_cpu
vmlinux 0xffffffff810c76f5
Build id: 3.10.0-693.el7.x86_64
0x85
39
dlm_tpool_tcb
dlm 0xffffffffc09fd126
Build id: 3.10.0-693.el7.x86_64
0x16
40
ofs_daemon_proxy
knal 0xffffffffc0ebf347
Build id: 3.10.0-693.el7.x86_64
0x67
41
? __schedule
vmlinux 0xffffffff816a8f8d
Build id: 3.10.0-693.el7.x86_64
0x39d
42
? ofs_thread_daemonize
knal 0xffffffffc0ebf2e0
Build id: 3.10.0-693.el7.x86_64
0x60
43
kthread
vmlinux 0xffffffff810b098f
Build id: 3.10.0-693.el7.x86_64
0xcf
44
? insert_kthread_work
vmlinux 0xffffffff810b08c0
Build id: 3.10.0-693.el7.x86_64
0x40
45
ret_from_fork
vmlinux 0xffffffff816b4f18
Build id: 3.10.0-693.el7.x86_64
0x58
46
? insert_kthread_work
vmlinux 0xffffffff810b08c0
Build id: 3.10.0-693.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [dlm_i_cb/0:9900]
Modules linked in: tun can_bcm can_raw can squashfs loop overlay(T) sctp_diag sctp dccp_diag dccp tcp_diag udp_diag inet_diag unix_diag ses enclosure fuse vfat fat isofs xfs uas usb_storage ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack libcrc32c iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter rctld(OE) parastor(OE) dlm(OE) kfnccli(OE) knal(OE) mmfs26(OE) mmfslinux(OE) tracedev(OE) rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_uverbs(OE)
 ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) iTCO_wdt iTCO_vendor_support skx_edac edac_core coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass joydev sg pcspkr shpchp i2c_i801 lpc_ich ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_cpufreq mei_me mei binfmt_misc nfsd knem(OE) auth_rpcgss nfs_acl lockd grace ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx5_ib(OE) ib_core(OE) 8021q garp stp crct10dif_pclmul llc crct10dif_common mrp crc32_pclmul crc32c_intel ghash_clmulni_intel i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt aesni_intel fb_sys_fops ttm lrw gf128mul glue_helper scsi_transport_iscsi ablk_helper drm cryptd ixgbe mpt3sas i40e raid_class ahci libahci dca mdio mlx5_core(OE) libata mlxfw(OE) mlx_compat(OE)
 devlink scsi_transport_sas ptp pps_core i2c_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod
CPU: 2 PID: 9900 Comm: dlm_i_cb/0 Tainted: G        W  OEL ------------ T 3.10.0-693.el7.x86_64 #1
Hardware name: Sugon I620-G30/60P24-US, BIOS 0JGST031 08/02/2018
task: ffff880aeef60fd0 ti: ffff880af6750000 task.ti: ffff880af6750000
RIP: 0010:[<ffffffffc1a56b81>]  [<ffffffffc1a56b81>] fsc_dentry_lru_del_init+0x31/0x100 [parastor]
RSP: 0018:ffff880af6753780  EFLAGS: 00000286
RAX: ffff8816937b01b8 RBX: ffff880554668a00 RCX: dead000000000200
RDX: ffff880d80eb0400 RSI: ffff8816937b0290 RDI: ffff880d80eb0300
RBP: ffff880af67537d0 R08: ffff880d80eb0400 R09: 0000000000000000
R10: ffff880554668a00 R11: ffff88051ba96610 R12: ffff88095a93b980
R13: ffff880af6753770 R14: ffff880af67536f8 R15: 0000000000000202
FS:  0000000000000000(0000) GS:ffff880c0f680000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f07c76f7000 CR3: 00000000019f2000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
 ffff880594078c00 ffff880594078c20 dead000000000200 ffff880594078c00
 ffff8816937b0290 ffff880594078c00 ffffffffffffff10 ffffffffc1a4cd27
 0000000000000010 ffff880d80eb0210 ffff880af6753850 ffffffffc1a4cd2f
Call Trace:
 [<ffffffffc1a4cd27>] ? fsc_shrink_dcache_parent+0xb7/0x180 [parastor]
 [<ffffffffc1a4cd2f>] fsc_shrink_dcache_parent+0xbf/0x180 [parastor]
 [<ffffffffc1a57f5c>] fscex_drop_negative_dentries+0x86c/0x910 [parastor]
 [<ffffffffc1b3214a>] ? fsc_storsys_find_bykey+0x3a/0xc0 [parastor]
 [<ffffffffc1b0b595>] ? cli_finode_xattr_reclaim+0x225/0x580 [parastor]
 [<ffffffffc1acdc8c>] cli_dlm_cncb+0x4dc/0x720 [parastor]
 [<ffffffffc0e2152a>] ? np_protoxtask_new+0x3a/0x370 [knal]
 [<ffffffffc1acd7b0>] ? cli_dlm_need_cncb_func+0x60/0x60 [parastor]
 [<ffffffffc0a1632e>] dlm_initr_call_cncb_do+0x13e/0x1c0 [dlm]
 [<ffffffffc0e1892d>] ? np_nrq_state_set.part.3+0x2d/0x80 [knal]
 [<ffffffffc0e1c420>] ? np_nrq_notifystop+0x70/0x70 [knal]
 [<ffffffffc0e1d6e0>] ? np_nrq_nalctrl_put+0x100/0x100 [knal]
 [<ffffffffc0e1d5e0>] ? np_protoxtask_final+0x100/0x100 [knal]
 [<ffffffffc0a17d88>] dlm_initr_call_cncb+0x2e8/0x470 [dlm]
 [<ffffffff810cd794>] ? update_curr+0x104/0x190
 [<ffffffff810ca29e>] ? account_entity_dequeue+0xae/0xd0
 [<ffffffff810cdc7c>] ? dequeue_entity+0x11c/0x5d0
 [<ffffffffc0e1d600>] ? np_nrq_nalctrl_put+0x20/0x100 [knal]
 [<ffffffffc0a10f15>] dlm_initr_call_blcb+0x2d5/0x350 [dlm]
 [<ffffffff810ce54e>] ? dequeue_task_fair+0x41e/0x660
 [<ffffffffc0a0021b>] ? dlm_lock_status_get+0x1b/0xd0 [dlm]
 [<ffffffff81029557>] ? __switch_to+0xd7/0x510
 [<ffffffffc0a11aab>] dlm_initr_callback+0xcb/0x2b0 [dlm]
 [<ffffffffc0a0014b>] ? dlm_lock_status_set+0x1b/0xd0 [dlm]
 [<ffffffffc0a0021b>] ? dlm_lock_status_get+0x1b/0xd0 [dlm]
 [<ffffffffc09f7b97>] ? dlm_memc_alloc+0x67/0x100 [dlm]
 [<ffffffffc0a14345>] ? dlm_initr_lock_lru_del+0xa5/0xf0 [dlm]
 [<ffffffffc0a12ec2>] dlm_initr_handle_cb+0xd22/0x1330 [dlm]
 [<ffffffff810b16d6>] ? finish_wait+0x56/0x70
 [<ffffffff810b1910>] ? wake_up_atomic_t+0x30/0x30
 [<ffffffffc0e1c690>] ? np_nrq_timeout_notify+0x270/0x270 [knal]
 [<ffffffffc0e1dab0>] ? nwatch_command_cmpkey+0x60/0x60 [knal]
 [<ffffffffc0e1db20>] ? np_module_nalctrl_final+0x30/0x30 [knal]
 [<ffffffffc0a0a485>] dlm_initr_cb_tcb+0x135/0x450 [dlm]
 [<ffffffff810cdc7c>] ? dequeue_entity+0x11c/0x5d0
 [<ffffffff810ce54e>] ? dequeue_task_fair+0x41e/0x660
 [<ffffffff810c76f5>] ? sched_clock_cpu+0x85/0xc0
 [<ffffffffc09fd126>] dlm_tpool_tcb+0x16/0x20 [dlm]
 [<ffffffffc0ebf347>] ofs_daemon_proxy+0x67/0x150 [knal]
 [<ffffffff816a8f8d>] ? __schedule+0x39d/0x8b0
 [<ffffffffc0ebf2e0>] ? ofs_thread_daemonize+0x60/0x60 [knal]
 [<ffffffff810b098f>] kthread+0xcf/0xe0
 [<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
 [<ffffffff816b4f18>] ret_from_fork+0x58/0x90
 [<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
Code: 48 89 e5 53 48 89 fb 48 83 ec 48 48 85 ff 0f 84 d6 00 00 00 48 8b 87 98 00 00 00 48 85 c0 0f 84 c4 00 00 00 48 8b 80 f8 00 00 00 <48> 85 c0 0f 84 b2 00 00 00 48 3b 3f 0f 84 91 00 00 00 83 a8 d4 
Complete report #327215
Frame # Function Binary Source or offset Line
1
__write_lock_failed
vmlinux 0xffffffff81331a89
Build id: 3.10.0-693.el7.x86_64
0x9
2
_raw_write_lock
vmlinux 0xffffffff816ab927
Build id: 3.10.0-693.el7.x86_64
0x17
3
fsc_d_drop
parastor 0xffffffffc1711147
Build id: 3.10.0-693.el7.x86_64
0x57
4
lcli_d_drop_fde
parastor 0xffffffffc1705bc4
Build id: 3.10.0-693.el7.x86_64
0xd4
5
lcli_d_release
parastor 0xffffffffc1705e0e
Build id: 3.10.0-693.el7.x86_64
0xbe
6
? inode_add_lru
vmlinux 0xffffffff8121ddcc
Build id: 3.10.0-693.el7.x86_64
0x3c
7
? iput
vmlinux 0xffffffff8121df73
Build id: 3.10.0-693.el7.x86_64
0x143
8
d_free
vmlinux 0xffffffff81218188
Build id: 3.10.0-693.el7.x86_64
0x38
9
shrink_dentry_list
vmlinux 0xffffffff8121a35f
Build id: 3.10.0-693.el7.x86_64
0x21f
10
shrink_dcache_parent
vmlinux 0xffffffff8121a6e8
Build id: 3.10.0-693.el7.x86_64
0x38
11
lcli_drop_negvde
parastor 0xffffffffc1706f36
Build id: 3.10.0-693.el7.x86_64
0x616
12
? blkgrp_vec_lookup
parastor 0xffffffffc17398ed
Build id: 3.10.0-693.el7.x86_64
0x1d
13
? truncate_inode_blkgrps_range_wait
parastor 0xffffffffc171ee5d
Build id: 3.10.0-693.el7.x86_64
0xcd
14
fscex_drop_negative_dentries
parastor 0xffffffffc17129fe
Build id: 3.10.0-693.el7.x86_64
0x30e
15
? __truncate_inode_blks_range
parastor 0xffffffffc171f495
Build id: 3.10.0-693.el7.x86_64
0xc5
16
? cli_dlm_cncb_drh2finode
parastor 0xffffffffc17879ab
Build id: 3.10.0-693.el7.x86_64
0x6b
17
cli_dlm_cncb
parastor 0xffffffffc1788c8c
Build id: 3.10.0-693.el7.x86_64
0x4dc
18
? do_gettimeofday
vmlinux 0xffffffff810ea9b9
Build id: 3.10.0-693.el7.x86_64
0x29
19
? cli_dlm_need_cncb_func
parastor 0xffffffffc17887b0
Build id: 3.10.0-693.el7.x86_64
0x60
20
dlm_initr_call_cncb_do
dlm 0xffffffffc08fe32e
Build id: 3.10.0-693.el7.x86_64
0x13e
21
? np_nrq_state_set.part.3
knal 0xffffffffc0ad392d
Build id: 3.10.0-693.el7.x86_64
0x2d
22
? np_nrq_notifystop
knal 0xffffffffc0ad7420
Build id: 3.10.0-693.el7.x86_64
0x70
23
? np_nrq_nalctrl_put
knal 0xffffffffc0ad86e0
Build id: 3.10.0-693.el7.x86_64
0x100
24
? np_protoxtask_final
knal 0xffffffffc0ad85e0
Build id: 3.10.0-693.el7.x86_64
0x100
25
dlm_initr_call_cncb
dlm 0xffffffffc08ffd88
Build id: 3.10.0-693.el7.x86_64
0x2e8
26
? update_curr
vmlinux 0xffffffff810cd794
Build id: 3.10.0-693.el7.x86_64
0x104
27
? account_entity_dequeue
vmlinux 0xffffffff810ca29e
Build id: 3.10.0-693.el7.x86_64
0xae
28
? dequeue_entity
vmlinux 0xffffffff810cdc7c
Build id: 3.10.0-693.el7.x86_64
0x11c
29
? np_nrq_nalctrl_put
knal 0xffffffffc0ad8600
Build id: 3.10.0-693.el7.x86_64
0x20
30
dlm_initr_call_blcb
dlm 0xffffffffc08f8f15
Build id: 3.10.0-693.el7.x86_64
0x2d5
31
? dequeue_task_fair
vmlinux 0xffffffff810ce54e
Build id: 3.10.0-693.el7.x86_64
0x41e
32
? dlm_lock_status_get
dlm 0xffffffffc08e821b
Build id: 3.10.0-693.el7.x86_64
0x1b
33
? __switch_to
vmlinux 0xffffffff81029557
Build id: 3.10.0-693.el7.x86_64
0xd7
34
dlm_initr_callback
dlm 0xffffffffc08f9aab
Build id: 3.10.0-693.el7.x86_64
0xcb
35
? dlm_lock_status_set
dlm 0xffffffffc08e814b
Build id: 3.10.0-693.el7.x86_64
0x1b
36
? dlm_lock_status_get
dlm 0xffffffffc08e821b
Build id: 3.10.0-693.el7.x86_64
0x1b
37
? dlm_memc_alloc
dlm 0xffffffffc08dfb97
Build id: 3.10.0-693.el7.x86_64
0x67
38
? dlm_initr_lock_lru_del
dlm 0xffffffffc08fc345
Build id: 3.10.0-693.el7.x86_64
0xa5
39
dlm_initr_handle_cb
dlm 0xffffffffc08faec2
Build id: 3.10.0-693.el7.x86_64
0xd22
40
? finish_wait
vmlinux 0xffffffff810b16d6
Build id: 3.10.0-693.el7.x86_64
0x56
41
? wake_up_atomic_t
vmlinux 0xffffffff810b1910
Build id: 3.10.0-693.el7.x86_64
0x30
42
? np_nrq_timeout_notify
knal 0xffffffffc0ad7690
Build id: 3.10.0-693.el7.x86_64
0x270
43
? nwatch_command_cmpkey
knal 0xffffffffc0ad8ab0
Build id: 3.10.0-693.el7.x86_64
0x60
44
? np_module_nalctrl_final
knal 0xffffffffc0ad8b20
Build id: 3.10.0-693.el7.x86_64
0x30
45
dlm_initr_cb_tcb
dlm 0xffffffffc08f2485
Build id: 3.10.0-693.el7.x86_64
0x135
46
? dequeue_entity
vmlinux 0xffffffff810cdc7c
Build id: 3.10.0-693.el7.x86_64
0x11c
47
? dequeue_task_fair
vmlinux 0xffffffff810ce54e
Build id: 3.10.0-693.el7.x86_64
0x41e
48
? sched_clock_cpu
vmlinux 0xffffffff810c76f5
Build id: 3.10.0-693.el7.x86_64
0x85
49
dlm_tpool_tcb
dlm 0xffffffffc08e5126
Build id: 3.10.0-693.el7.x86_64
0x16
50
ofs_daemon_proxy
knal 0xffffffffc0b7a347
Build id: 3.10.0-693.el7.x86_64
0x67
51
? __schedule
vmlinux 0xffffffff816a8f8d
Build id: 3.10.0-693.el7.x86_64
0x39d
52
? ofs_thread_daemonize
knal 0xffffffffc0b7a2e0
Build id: 3.10.0-693.el7.x86_64
0x60
53
kthread
vmlinux 0xffffffff810b098f
Build id: 3.10.0-693.el7.x86_64
0xcf
54
? insert_kthread_work
vmlinux 0xffffffff810b08c0
Build id: 3.10.0-693.el7.x86_64
0x40
55
ret_from_fork
vmlinux 0xffffffff816b4f18
Build id: 3.10.0-693.el7.x86_64
0x58
56
? insert_kthread_work
vmlinux 0xffffffff810b08c0
Build id: 3.10.0-693.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [dlm_i_cb/3:3410]
Modules linked in: ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables squashfs loop overlay(T) can_bcm sctp can_raw can iptable_filter mmfs26(OE) mmfslinux(OE) tracedev(OE) vfat fat isofs xfs libcrc32c uas usb_storage tcp_lp fuse rctld(OE) parastor(OE) dlm(OE) kfnccli(OE) knal(OE) rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_uverbs(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) iTCO_wdt
iTCO_vendor_support skx_edac edac_core coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass pcspkr joydev sg i2c_i801 lpc_ich mei_me shpchp mei ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_cpufreq nfsd auth_rpcgss nfs_acl knem(OE) lockd binfmt_misc grace ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx5_ib(OE) ib_core(OE) 8021q garp stp llc mrp crct10dif_pclmul crct10dif_common crc32_pclmul crc32c_intel scsi_transport_iscsi ghash_clmulni_intel i2c_algo_bit drm_kms_helper ahci libahci ixgbe mpt3sas syscopyarea sysfillrect sysimgblt fb_sys_fops aesni_intel ttm lrw gf128mul glue_helper ablk_helper drm cryptd i40e dca mdio raid_class libata mlx5_core(OE) scsi_transport_sas mlxfw(OE) mlx_compat(OE) devlink ptp pps_core i2c_core wmi sunrpc dm_mirror dm_region_hash
dm_log dm_mod [last unloaded: tracedev]
CPU: 5 PID: 3410 Comm: dlm_i_cb/3 Tainted: G        W  OE  ------------ T 3.10.0-693.el7.x86_64 #1
Hardware name: Sugon I620-G30/60P24-US, BIOS 0JGST031 08/02/2018
task: ffff8817f5f68fd0 ti: ffff8817404e8000 task.ti: ffff8817404e8000
RIP: 0010:[<ffffffff81331a89>]  [<ffffffff81331a89>] __write_lock_failed+0x9/0x20
RSP: 0018:ffff8817404eb570  EFLAGS: 00000297
RAX: ffff88173bb44000 RBX: 0000000000000010 RCX: dead000000000200
RDX: 0000000000000001 RSI: ffff88115f84a890 RDI: ffff880bb9f74294
RBP: ffff8817404eb570 R08: ffff88180a450450 R09: 000000018040001b
R10: ffff88180a4503c0 R11: ffffea0059542900 R12: ffff8817404eb550
R13: ffff8816550a6400 R14: ffff8816550a6410 R15: ffffffffc1710be7
FS:  0000000000000000(0000) GS:ffff88180da40000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00005559e46e0000 CR3: 00000003aa936000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffff8817404eb580 ffffffff816ab927 ffff8817404eb5f0 ffffffffc1711147
ffffffffffffff10 ffff880bb9f741b8 0000000000000010 0000000000000282
ffff8817404eb5c8 000000002d595303 ffff8817404eb700 ffff88180a450d80
Call Trace:
[<ffffffff816ab927>] _raw_write_lock+0x17/0x20
[<ffffffffc1711147>] fsc_d_drop+0x57/0x160 [parastor]
[<ffffffffc1705bc4>] lcli_d_drop_fde+0xd4/0x190 [parastor]
[<ffffffffc1705e0e>] lcli_d_release+0xbe/0xe0 [parastor]
[<ffffffff8121ddcc>] ? inode_add_lru+0x3c/0xa0
[<ffffffff8121df73>] ? iput+0x143/0x190
[<ffffffff81218188>] d_free+0x38/0x70
[<ffffffff8121a35f>] shrink_dentry_list+0x21f/0x490
[<ffffffff8121a6e8>] shrink_dcache_parent+0x38/0x90
[<ffffffffc1706f36>] lcli_drop_negvde+0x616/0x650 [parastor]
[<ffffffffc17398ed>] ? blkgrp_vec_lookup+0x1d/0x30 [parastor]
[<ffffffffc171ee5d>] ? truncate_inode_blkgrps_range_wait+0xcd/0x270 [parastor]
[<ffffffffc17129fe>] fscex_drop_negative_dentries+0x30e/0x910 [parastor]
[<ffffffffc171f495>] ? __truncate_inode_blks_range+0xc5/0xe0 [parastor]
[<ffffffffc17879ab>] ? cli_dlm_cncb_drh2finode+0x6b/0x180 [parastor]
[<ffffffffc1788c8c>] cli_dlm_cncb+0x4dc/0x720 [parastor]
[<ffffffff810ea9b9>] ? do_gettimeofday+0x29/0x70
[<ffffffffc17887b0>] ? cli_dlm_need_cncb_func+0x60/0x60 [parastor]
[<ffffffffc08fe32e>] dlm_initr_call_cncb_do+0x13e/0x1c0 [dlm]
[<ffffffffc0ad392d>] ? np_nrq_state_set.part.3+0x2d/0x80 [knal]
[<ffffffffc0ad7420>] ? np_nrq_notifystop+0x70/0x70 [knal]
[<ffffffffc0ad86e0>] ? np_nrq_nalctrl_put+0x100/0x100 [knal]
[<ffffffffc0ad85e0>] ? np_protoxtask_final+0x100/0x100 [knal]
[<ffffffffc08ffd88>] dlm_initr_call_cncb+0x2e8/0x470 [dlm]
[<ffffffff810cd794>] ? update_curr+0x104/0x190
[<ffffffff810ca29e>] ? account_entity_dequeue+0xae/0xd0
[<ffffffff810cdc7c>] ? dequeue_entity+0x11c/0x5d0
[<ffffffffc0ad8600>] ? np_nrq_nalctrl_put+0x20/0x100 [knal]
[<ffffffffc08f8f15>] dlm_initr_call_blcb+0x2d5/0x350 [dlm]
[<ffffffff810ce54e>] ? dequeue_task_fair+0x41e/0x660
[<ffffffffc08e821b>] ? dlm_lock_status_get+0x1b/0xd0 [dlm]
[<ffffffff81029557>] ? __switch_to+0xd7/0x510
[<ffffffffc08f9aab>] dlm_initr_callback+0xcb/0x2b0 [dlm]
[<ffffffffc08e814b>] ? dlm_lock_status_set+0x1b/0xd0 [dlm]
[<ffffffffc08e821b>] ? dlm_lock_status_get+0x1b/0xd0 [dlm]
[<ffffffffc08dfb97>] ? dlm_memc_alloc+0x67/0x100 [dlm]
[<ffffffffc08fc345>] ? dlm_initr_lock_lru_del+0xa5/0xf0 [dlm]
[<ffffffffc08faec2>] dlm_initr_handle_cb+0xd22/0x1330 [dlm]
[<ffffffff810b16d6>] ? finish_wait+0x56/0x70
[<ffffffff810b1910>] ? wake_up_atomic_t+0x30/0x30
[<ffffffffc0ad7690>] ? np_nrq_timeout_notify+0x270/0x270 [knal]
[<ffffffffc0ad8ab0>] ? nwatch_command_cmpkey+0x60/0x60 [knal]
[<ffffffffc0ad8b20>] ? np_module_nalctrl_final+0x30/0x30 [knal]
[<ffffffffc08f2485>] dlm_initr_cb_tcb+0x135/0x450 [dlm]
[<ffffffff810cdc7c>] ? dequeue_entity+0x11c/0x5d0
[<ffffffff810ce54e>] ? dequeue_task_fair+0x41e/0x660
[<ffffffff810c76f5>] ? sched_clock_cpu+0x85/0xc0
[<ffffffffc08e5126>] dlm_tpool_tcb+0x16/0x20 [dlm]
[<ffffffffc0b7a347>] ofs_daemon_proxy+0x67/0x150 [knal]
[<ffffffff816a8f8d>] ? __schedule+0x39d/0x8b0
[<ffffffffc0b7a2e0>] ? ofs_thread_daemonize+0x60/0x60 [knal]
[<ffffffff810b098f>] kthread+0xcf/0xe0
[<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
[<ffffffff816b4f18>] ret_from_fork+0x58/0x90
[<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
Code: 01 cb 48 89 01 31 c0 0f 01 ca c3 b8 f2 ff ff ff 0f 01 ca c3 90 90 90 90 90 90 90 90 90 90 90 90 90 90 55 48 89 e5 f0 ff 07 f3 90 <83> 3f 01 75 f9 f0 ff 0f 75 f1 5d c3 90 66 2e 0f 1f 84 00 00 00 
Complete report #139237
Frame # Function Binary Source or offset Line
1
fsc_shrink_dcache_parent
parastor 0xffffffffc179dd37
Build id: 3.10.0-693.el7.x86_64
0xb7
2
fscex_drop_negative_dentries
parastor 0xffffffffc17a935c
Build id: 3.10.0-693.el7.x86_64
0x86c
3
? fsc_storsys_find_bykey
parastor 0xffffffffc188564a
Build id: 3.10.0-693.el7.x86_64
0x3a
4
? cli_finode_xattr_reclaim
parastor 0xffffffffc185e605
Build id: 3.10.0-693.el7.x86_64
0x225
5
cli_dlm_cncb
parastor 0xffffffffc182092c
Build id: 3.10.0-693.el7.x86_64
0x4dc
6
? do_gettimeofday
vmlinux 0xffffffff810ea9b9
Build id: 3.10.0-693.el7.x86_64
0x29
7
? cli_dlm_need_cncb_func
parastor 0xffffffffc1820450
Build id: 3.10.0-693.el7.x86_64
0x60
8
dlm_initr_call_cncb_do
dlm 0xffffffffc157b51e
Build id: 3.10.0-693.el7.x86_64
0x13e
9
? np_nrq_state_set.part.3
knal 0xffffffffc095ed1d
Build id: 3.10.0-693.el7.x86_64
0x2d
10
? np_nrq_notifystop
knal 0xffffffffc0962880
Build id: 3.10.0-693.el7.x86_64
0x70
11
? np_nrq_nalctrl_put
knal 0xffffffffc0963b60
Build id: 3.10.0-693.el7.x86_64
0x100
12
? np_protoxtask_final
knal 0xffffffffc0963a60
Build id: 3.10.0-693.el7.x86_64
0x100
13
dlm_initr_call_cncb
dlm 0xffffffffc157cf78
Build id: 3.10.0-693.el7.x86_64
0x2e8
14
? update_curr
vmlinux 0xffffffff810cd794
Build id: 3.10.0-693.el7.x86_64
0x104
15
? account_entity_dequeue
vmlinux 0xffffffff810ca29e
Build id: 3.10.0-693.el7.x86_64
0xae
16
? dequeue_entity
vmlinux 0xffffffff810cdc7c
Build id: 3.10.0-693.el7.x86_64
0x11c
17
dlm_initr_call_blcb
dlm 0xffffffffc15760b5
Build id: 3.10.0-693.el7.x86_64
0x2d5
18
? dequeue_task_fair
vmlinux 0xffffffff810ce54e
Build id: 3.10.0-693.el7.x86_64
0x41e
19
? dlm_lock_status_get
dlm 0xffffffffc15651cb
Build id: 3.10.0-693.el7.x86_64
0x1b
20
? __switch_to
vmlinux 0xffffffff81029557
Build id: 3.10.0-693.el7.x86_64
0xd7
21
dlm_initr_callback
dlm 0xffffffffc1576c5b
Build id: 3.10.0-693.el7.x86_64
0xcb
22
? dlm_lock_status_set
dlm 0xffffffffc15650fb
Build id: 3.10.0-693.el7.x86_64
0x1b
23
? dlm_lock_status_get
dlm 0xffffffffc15651cb
Build id: 3.10.0-693.el7.x86_64
0x1b
24
? dlm_memc_alloc
dlm 0xffffffffc155cb9f
Build id: 3.10.0-693.el7.x86_64
0x6f
25
? dlm_initr_lock_lru_del
dlm 0xffffffffc1579525
Build id: 3.10.0-693.el7.x86_64
0xa5
26
dlm_initr_handle_cb
dlm 0xffffffffc1578072
Build id: 3.10.0-693.el7.x86_64
0xd22
27
? finish_wait
vmlinux 0xffffffff810b16d6
Build id: 3.10.0-693.el7.x86_64
0x56
28
? wake_up_atomic_t
vmlinux 0xffffffff810b1910
Build id: 3.10.0-693.el7.x86_64
0x30
29
? np_nrq_timeout_notify
knal 0xffffffffc0962af0
Build id: 3.10.0-693.el7.x86_64
0x270
30
? nwatch_command_cmpkey
knal 0xffffffffc0963f50
Build id: 3.10.0-693.el7.x86_64
0x60
31
? np_module_nalctrl_final
knal 0xffffffffc0963fd0
Build id: 3.10.0-693.el7.x86_64
0x30
32
dlm_initr_cb_tcb
dlm 0xffffffffc156f505
Build id: 3.10.0-693.el7.x86_64
0x135
33
? dequeue_entity
vmlinux 0xffffffff810cdc7c
Build id: 3.10.0-693.el7.x86_64
0x11c
34
? dequeue_task_fair
vmlinux 0xffffffff810ce54e
Build id: 3.10.0-693.el7.x86_64
0x41e
35
? sched_clock_cpu
vmlinux 0xffffffff810c76f5
Build id: 3.10.0-693.el7.x86_64
0x85
36
dlm_tpool_tcb
dlm 0xffffffffc1562126
Build id: 3.10.0-693.el7.x86_64
0x16
37
ofs_daemon_proxy
knal 0xffffffffc0a06367
Build id: 3.10.0-693.el7.x86_64
0x67
38
? __schedule
vmlinux 0xffffffff816a8f8d
Build id: 3.10.0-693.el7.x86_64
0x39d
39
? ofs_thread_daemonize
knal 0xffffffffc0a06300
Build id: 3.10.0-693.el7.x86_64
0x60
40
kthread
vmlinux 0xffffffff810b098f
Build id: 3.10.0-693.el7.x86_64
0xcf
41
? insert_kthread_work
vmlinux 0xffffffff810b08c0
Build id: 3.10.0-693.el7.x86_64
0x40
42
ret_from_fork
vmlinux 0xffffffff816b4f18
Build id: 3.10.0-693.el7.x86_64
0x58
43
? insert_kthread_work
vmlinux 0xffffffff810b08c0
Build id: 3.10.0-693.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#8 stuck for 22s! [dlm_i_cb/0:5164]
Modules linked in: nfsv3 rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache ps200b(OE) parastor(OE) dlm(OE) kfnccli(OE) knal(OE) nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter overlay(T) rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_uverbs(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx5_ib(OE) mlx5_core(OE) mlxfw(OE) mlx4_en(OE) dm_mirror dm_region_hash dm_log dm_mod sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel
lrw gf128mul iTCO_wdt joydev glue_helper ablk_helper ipmi_si iTCO_vendor_support cryptd ipmi_devintf ipmi_msghandler pcspkr sg shpchp i2c_i801 mei_me mei wmi lpc_ich nfsd auth_rpcgss nfs_acl lockd grace sunrpc knem(OE) ip_tables ext4 mbcache jbd2 mlx4_ib(OE) ib_core(OE) sd_mod sr_mod cdrom crc_t10dif crct10dif_generic ast drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm drm libsas ahci igb crct10dif_pclmul e1000e scsi_transport_sas libahci crct10dif_common crc32c_intel libata mlx4_core(OE) devlink megaraid_sas mlx_compat(OE) dca ptp i2c_algo_bit pps_core i2c_core
CPU: 8 PID: 5164 Comm: dlm_i_cb/0 Tainted: G           OEL ------------ T 3.10.0-693.el7.x86_64 #1
Hardware name: empty empty/S7056 , BIOS V7.123 12/02/2013
task: ffff8808323f0fd0 ti: ffff88077d548000 task.ti: ffff88077d548000
RIP: 0010:[<ffffffffc179dd37>]  [<ffffffffc179dd37>] fsc_shrink_dcache_parent+0xb7/0x190 [parastor]
RSP: 0018:ffff88077d54b7e0  EFLAGS: 00000287
RAX: ffff8801ba2a0320 RBX: 0000000000000010 RCX: dead000000000200
RDX: ffff8801ba2a0300 RSI: ffff88085dbac290 RDI: ffff8801ba2a0300
RBP: ffff88077d54b850 R08: ffff8801ba2a0300 R09: ffff88045df48880
R10: ffff8805cdf11d00 R11: ffff8805b15bcc10 R12: ffffffffffffff10
R13: ffff8807d1d61000 R14: ffff88085dbac290 R15: ffff8807d1d61000
FS:  0000000000000000(0000) GS:ffff88087f680000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f7474c4e000 CR3: 00000000019f2000 CR4: 00000000001407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff88048276f100 ffff8803da384000 ffff8808323f0fd0 ffff8805cdf11d20
ffff88085dbac290 ffff88085dbac1b8 000000004341c000 ffff8805cdf11d00
ffff88085dbac290 000000000001b463 0000000000001321 ffff88024341bfc0
Call Trace:
[<ffffffffc17a935c>] fscex_drop_negative_dentries+0x86c/0x920 [parastor]
[<ffffffffc188564a>] ? fsc_storsys_find_bykey+0x3a/0xc0 [parastor]
[<ffffffffc185e605>] ? cli_finode_xattr_reclaim+0x225/0x590 [parastor]
[<ffffffffc182092c>] cli_dlm_cncb+0x4dc/0x740 [parastor]
[<ffffffff810ea9b9>] ? do_gettimeofday+0x29/0x70
[<ffffffffc1820450>] ? cli_dlm_need_cncb_func+0x60/0x60 [parastor]
[<ffffffffc157b51e>] dlm_initr_call_cncb_do+0x13e/0x1c0 [dlm]
[<ffffffffc095ed1d>] ? np_nrq_state_set.part.3+0x2d/0x80 [knal]
[<ffffffffc0962880>] ? np_nrq_notifystop+0x70/0x70 [knal]
[<ffffffffc0963b60>] ? np_nrq_nalctrl_put+0x100/0x100 [knal]
[<ffffffffc0963a60>] ? np_protoxtask_final+0x100/0x100 [knal]
[<ffffffffc157cf78>] dlm_initr_call_cncb+0x2e8/0x470 [dlm]
[<ffffffff810cd794>] ? update_curr+0x104/0x190
[<ffffffff810ca29e>] ? account_entity_dequeue+0xae/0xd0
[<ffffffff810cdc7c>] ? dequeue_entity+0x11c/0x5d0
[<ffffffffc15760b5>] dlm_initr_call_blcb+0x2d5/0x350 [dlm]
[<ffffffff810ce54e>] ? dequeue_task_fair+0x41e/0x660
[<ffffffffc15651cb>] ? dlm_lock_status_get+0x1b/0xd0 [dlm]
[<ffffffff81029557>] ? __switch_to+0xd7/0x510
[<ffffffffc1576c5b>] dlm_initr_callback+0xcb/0x2b0 [dlm]
[<ffffffffc15650fb>] ? dlm_lock_status_set+0x1b/0xd0 [dlm]
[<ffffffffc15651cb>] ? dlm_lock_status_get+0x1b/0xd0 [dlm]
[<ffffffffc155cb9f>] ? dlm_memc_alloc+0x6f/0x100 [dlm]
[<ffffffffc1579525>] ? dlm_initr_lock_lru_del+0xa5/0xf0 [dlm]
[<ffffffffc1578072>] dlm_initr_handle_cb+0xd22/0x1350 [dlm]
[<ffffffff810b16d6>] ? finish_wait+0x56/0x70
[<ffffffff810b1910>] ? wake_up_atomic_t+0x30/0x30
[<ffffffffc0962af0>] ? np_nrq_timeout_notify+0x270/0x270 [knal]
[<ffffffffc0963f50>] ? nwatch_command_cmpkey+0x60/0x60 [knal]
[<ffffffffc0963fd0>] ? np_module_nalctrl_final+0x30/0x30 [knal]
[<ffffffffc156f505>] dlm_initr_cb_tcb+0x135/0x480 [dlm]
[<ffffffff810cdc7c>] ? dequeue_entity+0x11c/0x5d0
[<ffffffff810ce54e>] ? dequeue_task_fair+0x41e/0x660
[<ffffffff810c76f5>] ? sched_clock_cpu+0x85/0xc0
[<ffffffffc1562126>] dlm_tpool_tcb+0x16/0x20 [dlm]
[<ffffffffc0a06367>] ofs_daemon_proxy+0x67/0x150 [knal]
[<ffffffff816a8f8d>] ? __schedule+0x39d/0x8b0
[<ffffffffc0a06300>] ? ofs_thread_daemonize+0x60/0x60 [knal]
[<ffffffff810b098f>] kthread+0xcf/0xe0
[<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
[<ffffffff816b4f18>] ret_from_fork+0x58/0x90
[<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
Code: c8 c7 45 c4 00 00 00 00 48 63 80 a4 04 00 00 48 89 83 e0 00 00 00 4d 8b 7d 20 eb 2c 0f 1f 84 00 00 00 00 00 4d 8d 67 f0 49 8b 1f <4c> 89 e7 e8 b1 a1 00 00 41 8b 47 68 85 c0 74 29 49 8d 47 10 49