# zpool replace p2 /dev/s3-1


INFO: task txg_sync:5338 blocked for more than 120 seconds.
      Tainted: P           O    4.14.24 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
txg_sync        D    0  5338      2 0x80000080
Call Trace:
 __schedule+0x2e1/0x730
 ? __dbuf_hold_impl+0x15a/0x5b0 [zfs]
 ? spl_kmem_alloc+0x96/0x1a0 [spl]
 schedule+0x3a/0xa0
 ? down_read+0x16/0x40
 ? mutex_lock+0x16/0x40
 schedule_preempt_disabled+0xe/0x10
 __mutex_lock.clone.1+0x2b8/0x590
 ? tsd_hash_search+0x4d/0xc0 [spl]
 ? dbuf_cache_target_bytes+0x28/0x40 [zfs]
 ? mutex_lock+0x16/0x40
 ? vdev_resilver_needed+0x125/0x1a0 [zfs]
 __mutex_lock_slowpath+0x17/0x20
 mutex_lock+0x33/0x40
 spa_man_trim_stop+0x64/0xd0 [zfs]
 dsl_scan_scrubbing+0x1d/0x40 [zfs]
 dsl_scan_is_paused_scrub+0x19/0x40 [zfs]
 spa_scan_stat_init+0x40/0x90 [zfs]
 dsl_scan_setup_sync+0xd7/0x230 [zfs]
 dsl_scan_sync+0xd5/0x910 [zfs]
 ? zio_wait+0x13b/0x180 [zfs]
 ? ddt_sync+0x92/0xb0 [zfs]
 spa_sync+0x4d1/0x9d0 [zfs]
 ? woken_wake_function+0x20/0x20
 txg_sync_thread+0x2ca/0x340 [zfs]
 ? txg_dispatch_callbacks+0x1a0/0x1a0 [zfs]
 thread_generic_wrapper+0x79/0x90 [spl]
 ? __thread_create+0x150/0x150 [spl]
 kthread+0x111/0x150
 ? __kthread_init_worker+0x40/0x40
 ? __kthread_init_worker+0x40/0x40
 ret_from_fork+0x35/0x40
INFO: task zpool:9257 blocked for more than 120 seconds.
      Tainted: P           O    4.14.24 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
zpool           D    0  9257   5465 0x00000080
Call Trace:
 __schedule+0x2e1/0x730
 ? __wake_up_common_lock+0x80/0xa0
 schedule+0x3a/0xa0
 cv_wait_common+0x128/0x150 [spl]
 ? woken_wake_function+0x20/0x20
 __cv_wait+0x15/0x20 [spl]
 txg_wait_synced+0xe9/0x140 [zfs]
 spa_vdev_config_exit+0x107/0x120 [zfs]
 spa_vdev_exit+0x2b/0x140 [zfs]
 spa_vdev_attach+0x32a/0x520 [zfs]
 ? spl_vmem_free+0xe/0x10 [spl]
 zfs_ioc_vdev_attach+0x96/0xb0 [zfs]
 zfsdev_ioctl+0x5c1/0x600 [zfs]
 ? find_vma+0x20/0x80
 vfs_ioctl+0x18/0x30
 do_vfs_ioctl+0x153/0x420
 ? syscall_trace_enter+0x204/0x2e0
 SyS_ioctl+0x92/0xa0
 ? do_page_fault+0x46/0x130
 do_syscall_64+0x7d/0x160
 ? syscall_return_slowpath+0x90/0xc0
 ? schedule_tail+0x56/0xc0
 entry_SYSCALL_64_after_hwframe+0x3d/0xa2
RIP: 0033:0x35c60e0b77
RSP: 002b:00007ffca0e19858 EFLAGS: 00000246 ORIG_RAX: 0000000000000010
RAX: ffffffffffffffda RBX: 00000000012f9e70 RCX: 00000035c60e0b77
RDX: 00007ffca0e198d0 RSI: 0000000000005a0e RDI: 0000000000000003
RBP: 00007ffca0e198d0 R08: 0000000000000020 R09: 6968736100000008
R10: 00007ffca0e195e0 R11: 0000000000000246 R12: 000000000130eda8
R13: 00007ffca0e1ce80 R14: 0000000000000001 R15: 00000000012f9060
INFO: task zpool:9334 blocked for more than 120 seconds.
      Tainted: P           O    4.14.24 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
zpool           D    0  9334   9332 0x00000080
Call Trace:
 __schedule+0x2e1/0x730
 ? __pagevec_lru_add_fn+0x16a/0x2f0
 schedule+0x3a/0xa0
 schedule_preempt_disabled+0xe/0x10
 __mutex_lock.clone.1+0x2b8/0x590
 ? spl_kmem_alloc_impl+0x8f/0x1a0 [spl]
 ? spl_vmem_alloc+0x19/0x20 [spl]
 ? nv_alloc_sleep_spl+0x1f/0x30 [znvpair]
 ? nv_mem_zalloc+0x38/0x50 [znvpair]
 __mutex_lock_slowpath+0x17/0x20
 mutex_lock+0x33/0x40
 spa_all_configs+0x42/0x140 [zfs]
 zfs_ioc_pool_configs+0x25/0x90 [zfs]
 zfsdev_ioctl+0x5c1/0x600 [zfs]
 ? find_vma+0x20/0x80
 vfs_ioctl+0x18/0x30
 do_vfs_ioctl+0x153/0x420
 ? syscall_trace_enter+0x204/0x2e0
 SyS_ioctl+0x92/0xa0
 ? do_page_fault+0x46/0x130
 do_syscall_64+0x7d/0x160
 ? prepare_exit_to_usermode+0x8b/0xa0
 entry_SYSCALL_64_after_hwframe+0x3d/0xa2
RIP: 0033:0x35c60e0b77
RSP: 002b:00007ffc6b569a68 EFLAGS: 00000202 ORIG_RAX: 0000000000000010
RAX: ffffffffffffffda RBX: 0000000001a0b060 RCX: 00000035c60e0b77
RDX: 00007ffc6b569a90 RSI: 0000000000005a04 RDI: 0000000000000003
RBP: 00007ffc6b569a90 R08: 0000000000000008 R09: 0000000000200000
R10: 00007ffc6b5697f0 R11: 0000000000000202 R12: 00007ffc6b56d18c
R13: 00000000004063c0 R14: 0000000001a0be70 R15: 00007ffc6b56d130
INFO: task zpool:9352 blocked for more than 120 seconds.
      Tainted: P           O    4.14.24 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
zpool           D    0  9352   9346 0x00000080
Call Trace:
 __schedule+0x2e1/0x730
 schedule+0x3a/0xa0
 ? cache_grow_begin+0x65/0x260
 schedule_preempt_disabled+0xe/0x10
 __mutex_lock.clone.1+0x2b8/0x590
 ? spl_kmem_alloc_impl+0x8f/0x1a0 [spl]
 ? spl_vmem_alloc+0x19/0x20 [spl]
 ? nv_alloc_sleep_spl+0x1f/0x30 [znvpair]
 ? nv_mem_zalloc+0x38/0x50 [znvpair]
 __mutex_lock_slowpath+0x17/0x20
 mutex_lock+0x33/0x40
 spa_all_configs+0x42/0x140 [zfs]
 zfs_ioc_pool_configs+0x25/0x90 [zfs]
 zfsdev_ioctl+0x5c1/0x600 [zfs]
 ? find_vma+0x20/0x80
 vfs_ioctl+0x18/0x30
 do_vfs_ioctl+0x153/0x420
 ? syscall_trace_enter+0x204/0x2e0
 SyS_ioctl+0x92/0xa0
 ? do_page_fault+0x46/0x130
 do_syscall_64+0x7d/0x160
 ? syscall_return_slowpath+0x90/0xc0
 ? schedule_tail+0x56/0xc0
 entry_SYSCALL_64_after_hwframe+0x3d/0xa2
RIP: 0033:0x35c60e0b77
RSP: 002b:00007fffdf076fa8 EFLAGS: 00000202 ORIG_RAX: 0000000000000010
RAX: ffffffffffffffda RBX: 0000000000acf060 RCX: 00000035c60e0b77
RDX: 00007fffdf076fd0 RSI: 0000000000005a04 RDI: 0000000000000003
RBP: 00007fffdf076fd0 R08: 0000000000000008 R09: 0000000000200000
R10: 00007fffdf076d30 R11: 0000000000000202 R12: 00007fffdf07a65c
R13: 00000000004063c0 R14: 0000000000ad00f0 R15: 00007fffdf07a640
INFO: task zfs:9493 blocked for more than 120 seconds.
      Tainted: P           O    4.14.24 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
zfs             D    0  9493   9492 0x00000080
Call Trace:
 __schedule+0x2e1/0x730
 ? kmem_getpages+0x90/0x210
 schedule+0x3a/0xa0
 schedule_preempt_disabled+0xe/0x10
 __mutex_lock.clone.1+0x2b8/0x590
 ? mem_cgroup_commit_charge+0x96/0x130
 ? alloc_pages_vma+0x89/0x180
 ? lru_cache_add_active_or_unevictable+0x41/0xd0
 __mutex_lock_slowpath+0x17/0x20
 mutex_lock+0x33/0x40
 spa_open_common+0x63/0x4a0 [zfs]
 ? entity_namecheck+0xc4/0x210 [zcommon]
 spa_open+0x13/0x20 [zfs]
 pool_status_check+0x3c/0xd0 [zfs]
 zfsdev_ioctl+0x24c/0x600 [zfs]
 ? find_vma+0x20/0x80
 vfs_ioctl+0x18/0x30
 do_vfs_ioctl+0x153/0x420
 ? syscall_trace_enter+0x204/0x2e0
 SyS_ioctl+0x92/0xa0
 ? do_page_fault+0x46/0x130
 do_syscall_64+0x7d/0x160
 ? prepare_exit_to_usermode+0x8b/0xa0
 entry_SYSCALL_64_after_hwframe+0x3d/0xa2
RIP: 0033:0x35c60e0b77
RSP: 002b:00007ffffb560fb8 EFLAGS: 00000206 ORIG_RAX: 0000000000000010
RAX: ffffffffffffffda RBX: 00007ffffb560fe0 RCX: 00000035c60e0b77
RDX: 00007ffffb560fe0 RSI: 0000000000005a12 RDI: 0000000000000003
RBP: 00000000008b4060 R08: 0000000000000008 R09: 0000000000200000
R10: 00007ffffb560d40 R11: 0000000000000206 R12: 00007ffffb566f10
R13: 00000000008b63e0 R14: 00007ffffb564f50 R15: 00007ffffb5645d0