Thread 614 (LWP 102944 of process 80572):
#0  _umtx_op_err () at /usr/home/mmacy/devel/networking/lib/libthr/arch/amd64/amd64/_umtx_op_err.S:37
#1  0x0000000800687571 in __thr_rwlock_rdlock (rwlock=0x800ddc000, flags=<optimized out>, tsp=<optimized out>) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_umtx.c:307
#2  0x000000080068b449 in rwlock_rdlock_common (rwlock=<optimized out>, abstime=0x0) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_rwlock.c:206
#3  0x0000000000212395 in ztest_spa_prop_get_set (zd=0x800dce700, id=22) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:5469
#4  0x000000000022571e in ztest_execute (test=13, zi=0x2271e0 <ztest_info+416>, id=22) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6539
#5  0x0000000000224ff9 in ztest_thread (arg=0x16) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6586
#6  0x0000000800685426 in thread_start (curthread=0x80dfed400) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_create.c:291
#7  0x0000000000000000 in ?? ()
Backtrace stopped: Cannot access memory at address 0x7fff92d96000

Thread 613 (LWP 102943 of process 80572):
#0  _umtx_op_err () at /usr/home/mmacy/devel/networking/lib/libthr/arch/amd64/amd64/_umtx_op_err.S:37
#1  0x00000008006875c0 in __thr_rwlock_wrlock (rwlock=0x800ddc000, tsp=<optimized out>) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_umtx.c:327
#2  0x000000080068b6ab in rwlock_wrlock_common (rwlock=<optimized out>, abstime=0x0) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_rwlock.c:314
#3  0x000000000021370a in ztest_ddt_repair (zd=0x800dccbc0, id=21) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:5933
#4  0x000000000022571e in ztest_execute (test=18, zi=0x227280 <ztest_info+576>, id=21) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6539
#5  0x0000000000224ff9 in ztest_thread (arg=0x15) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6586
#6  0x0000000800685426 in thread_start (curthread=0x80dfecf00) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_create.c:291
#7  0x0000000000000000 in ?? ()
Backtrace stopped: Cannot access memory at address 0x7fff92f97000

Thread 612 (LWP 102942 of process 80572):
#0  _umtx_op_err () at /usr/home/mmacy/devel/networking/lib/libthr/arch/amd64/amd64/_umtx_op_err.S:37
#1  0x0000000800687571 in __thr_rwlock_rdlock (rwlock=0x800ddc000, flags=<optimized out>, tsp=<optimized out>) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_umtx.c:307
#2  0x000000080068b449 in rwlock_rdlock_common (rwlock=<optimized out>, abstime=0x0) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_rwlock.c:206
#3  0x0000000000212218 in ztest_dsl_prop_get_set (zd=0x800dd6f40, id=20) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:5437
#4  0x000000000022571e in ztest_execute (test=12, zi=0x2271c0 <ztest_info+384>, id=20) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6539
#5  0x0000000000224ff9 in ztest_thread (arg=0x14) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6586
#6  0x0000000800685426 in thread_start (curthread=0x80dfeca00) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_create.c:291
#7  0x0000000000000000 in ?? ()
Backtrace stopped: Cannot access memory at address 0x7fff93198000

Thread 611 (LWP 102941 of process 80572):
#0  _umtx_op_err () at /usr/home/mmacy/devel/networking/lib/libthr/arch/amd64/amd64/_umtx_op_err.S:37
#1  0x0000000800687571 in __thr_rwlock_rdlock (rwlock=0x800ddc000, flags=<optimized out>, tsp=<optimized out>) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_umtx.c:307
#2  0x000000080068b449 in rwlock_rdlock_common (rwlock=<optimized out>, abstime=0x0) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_rwlock.c:206
#3  0x0000000000212218 in ztest_dsl_prop_get_set (zd=0x800dd5400, id=19) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:5437
#4  0x000000000022571e in ztest_execute (test=12, zi=0x2271c0 <ztest_info+384>, id=19) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6539
#5  0x0000000000224ff9 in ztest_thread (arg=0x13) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6586
#6  0x0000000800685426 in thread_start (curthread=0x80dfec500) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_create.c:291
#7  0x0000000000000000 in ?? ()
Backtrace stopped: Cannot access memory at address 0x7fff93399000

Thread 610 (LWP 102940 of process 80572):
#0  _umtx_op_err () at /usr/home/mmacy/devel/networking/lib/libthr/arch/amd64/amd64/_umtx_op_err.S:37
#1  0x0000000800687571 in __thr_rwlock_rdlock (rwlock=0x800ddc000, flags=<optimized out>, tsp=<optimized out>) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_umtx.c:307
#2  0x000000080068b449 in rwlock_rdlock_common (rwlock=<optimized out>, abstime=0x0) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_rwlock.c:206
#3  0x00000000002127c2 in ztest_dmu_snapshot_create_destroy (zd=0x800dd38c0, id=18) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:4043
#4  0x000000000022571e in ztest_execute (test=15, zi=0x227220 <ztest_info+480>, id=18) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6539
#5  0x0000000000224ff9 in ztest_thread (arg=0x12) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6586
#6  0x0000000800685426 in thread_start (curthread=0x80dfec000) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_create.c:291
#7  0x0000000000000000 in ?? ()
Backtrace stopped: Cannot access memory at address 0x7fff9359a000

Thread 609 (LWP 102938 of process 80572):
#0  _umtx_op_err () at /usr/home/mmacy/devel/networking/lib/libthr/arch/amd64/amd64/_umtx_op_err.S:37
#1  0x0000000800687571 in __thr_rwlock_rdlock (rwlock=0x800ddc000, flags=<optimized out>, tsp=<optimized out>) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_umtx.c:307
#2  0x000000080068b449 in rwlock_rdlock_common (rwlock=<optimized out>, abstime=0x0) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_rwlock.c:206
#3  0x0000000000212218 in ztest_dsl_prop_get_set (zd=0x800dd1d80, id=17) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:5437
#4  0x000000000022571e in ztest_execute (test=12, zi=0x2271c0 <ztest_info+384>, id=17) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6539
#5  0x0000000000224ff9 in ztest_thread (arg=0x11) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6586
#6  0x0000000800685426 in thread_start (curthread=0x80df6eb00) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_create.c:291
#7  0x0000000000000000 in ?? ()
Backtrace stopped: Cannot access memory at address 0x7fff9379b000

Thread 608 (LWP 102937 of process 80572):
#0  _umtx_op_err () at /usr/home/mmacy/devel/networking/lib/libthr/arch/amd64/amd64/_umtx_op_err.S:37
#1  0x0000000800687571 in __thr_rwlock_rdlock (rwlock=0x800ddc000, flags=<optimized out>, tsp=<optimized out>) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_umtx.c:307
#2  0x000000080068b449 in rwlock_rdlock_common (rwlock=<optimized out>, abstime=0x0) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_rwlock.c:206
#3  0x0000000000212218 in ztest_dsl_prop_get_set (zd=0x800dd0240, id=16) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:5437
#4  0x000000000022571e in ztest_execute (test=12, zi=0x2271c0 <ztest_info+384>, id=16) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6539
#5  0x0000000000224ff9 in ztest_thread (arg=0x10) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6586
#6  0x0000000800685426 in thread_start (curthread=0x80df6e600) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_create.c:291
#7  0x0000000000000000 in ?? ()
Backtrace stopped: Cannot access memory at address 0x7fff9399c000

Thread 607 (LWP 102936 of process 80572):
#0  _umtx_op_err () at /usr/home/mmacy/devel/networking/lib/libthr/arch/amd64/amd64/_umtx_op_err.S:37
#1  0x0000000800687571 in __thr_rwlock_rdlock (rwlock=0x800ddc000, flags=<optimized out>, tsp=<optimized out>) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_umtx.c:307
#2  0x000000080068b449 in rwlock_rdlock_common (rwlock=<optimized out>, abstime=0x0) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_rwlock.c:206
#3  0x000000000021bef1 in ztest_io (zd=0x800dce700, object=183, offset=35184590192640) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:2589
#4  0x000000000020e14a in ztest_dmu_write_parallel (zd=0x800dce700, id=15) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:4817
#5  0x000000000022571e in ztest_execute (test=1, zi=0x227060 <ztest_info+32>, id=15) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6539
#6  0x0000000000224ff9 in ztest_thread (arg=0xf) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6586
#7  0x0000000800685426 in thread_start (curthread=0x80df6e100) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_create.c:291
#8  0x0000000000000000 in ?? ()
Backtrace stopped: Cannot access memory at address 0x7fff93b9d000

Thread 606 (LWP 102933 of process 80572):
#0  _umtx_op_err () at /usr/home/mmacy/devel/networking/lib/libthr/arch/amd64/amd64/_umtx_op_err.S:37
#1  0x0000000800687571 in __thr_rwlock_rdlock (rwlock=0x800ddc000, flags=<optimized out>, tsp=<optimized out>) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_umtx.c:307
#2  0x000000080068b449 in rwlock_rdlock_common (rwlock=<optimized out>, abstime=0x0) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_rwlock.c:206
#3  0x0000000000211c20 in ztest_dmu_objset_create_destroy (zd=0x800dccbc0, id=14) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:3942
#4  0x000000000022571e in ztest_execute (test=11, zi=0x2271a0 <ztest_info+352>, id=14) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6539
#5  0x0000000000224ff9 in ztest_thread (arg=0xe) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6586
#6  0x0000000800685426 in thread_start (curthread=0x80df01b00) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_create.c:291
#7  0x0000000000000000 in ?? ()
Backtrace stopped: Cannot access memory at address 0x7fff93d9e000

Thread 605 (LWP 102932 of process 80572):
#0  _umtx_op_err () at /usr/home/mmacy/devel/networking/lib/libthr/arch/amd64/amd64/_umtx_op_err.S:37
#1  0x0000000800687571 in __thr_rwlock_rdlock (rwlock=0x800ddc000, flags=<optimized out>, tsp=<optimized out>) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_umtx.c:307
#2  0x000000080068b449 in rwlock_rdlock_common (rwlock=<optimized out>, abstime=0x0) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_rwlock.c:206
#3  0x0000000000211c20 in ztest_dmu_objset_create_destroy (zd=0x800dd6f40, id=13) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:3942
#4  0x000000000022571e in ztest_execute (test=11, zi=0x2271a0 <ztest_info+352>, id=13) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6539
#5  0x0000000000224ff9 in ztest_thread (arg=0xd) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6586
#6  0x0000000800685426 in thread_start (curthread=0x80df00c00) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_create.c:291
#7  0x0000000000000000 in ?? ()
Backtrace stopped: Cannot access memory at address 0x7fff93f9f000

Thread 604 (LWP 102931 of process 80572):
#0  _umtx_op_err () at /usr/home/mmacy/devel/networking/lib/libthr/arch/amd64/amd64/_umtx_op_err.S:37
#1  0x0000000800687571 in __thr_rwlock_rdlock (rwlock=0x800ddc000, flags=<optimized out>, tsp=<optimized out>) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_umtx.c:307
#2  0x000000080068b449 in rwlock_rdlock_common (rwlock=<optimized out>, abstime=0x0) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_rwlock.c:206
#3  0x0000000000211c20 in ztest_dmu_objset_create_destroy (zd=0x800dd5400, id=12) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:3942
#4  0x000000000022571e in ztest_execute (test=11, zi=0x2271a0 <ztest_info+352>, id=12) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6539
#5  0x0000000000224ff9 in ztest_thread (arg=0xc) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6586
#6  0x0000000800685426 in thread_start (curthread=0x80df00700) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_create.c:291
#7  0x0000000000000000 in ?? ()
Backtrace stopped: Cannot access memory at address 0x7fff941a0000

Thread 603 (LWP 102930 of process 80572):
#0  _umtx_op_err () at /usr/home/mmacy/devel/networking/lib/libthr/arch/amd64/amd64/_umtx_op_err.S:37
#1  0x0000000800687571 in __thr_rwlock_rdlock (rwlock=0x800ddc000, flags=<optimized out>, tsp=<optimized out>) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_umtx.c:307
#2  0x000000080068b449 in rwlock_rdlock_common (rwlock=<optimized out>, abstime=0x0) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_rwlock.c:206
#3  0x0000000000211c20 in ztest_dmu_objset_create_destroy (zd=0x800dd38c0, id=11) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:3942
#4  0x000000000022571e in ztest_execute (test=11, zi=0x2271a0 <ztest_info+352>, id=11) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6539
#5  0x0000000000224ff9 in ztest_thread (arg=0xb) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6586
#6  0x0000000800685426 in thread_start (curthread=0x80df00200) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_create.c:291
#7  0x0000000000000000 in ?? ()
Backtrace stopped: Cannot access memory at address 0x7fff943a1000

Thread 600 (LWP 102924 of process 80572):
#0  _umtx_op_err () at /usr/home/mmacy/devel/networking/lib/libthr/arch/amd64/amd64/_umtx_op_err.S:37
#1  0x0000000800687571 in __thr_rwlock_rdlock (rwlock=0x800ddc000, flags=<optimized out>, tsp=<optimized out>) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_umtx.c:307
#2  0x000000080068b449 in rwlock_rdlock_common (rwlock=<optimized out>, abstime=0x0) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_rwlock.c:206
#3  0x0000000000213e38 in ztest_dmu_snapshot_hold (zd=0x800dce700, id=8) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:5516
#4  0x000000000022571e in ztest_execute (test=19, zi=0x2272a0 <ztest_info+608>, id=8) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6539
#5  0x0000000000224ff9 in ztest_thread (arg=0x8) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6586
#6  0x0000000800685426 in thread_start (curthread=0x80358ef00) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_create.c:291
#7  0x0000000000000000 in ?? ()
Backtrace stopped: Cannot access memory at address 0x7fff949a4000

Thread 599 (LWP 102923 of process 80572):
#0  _umtx_op_err () at /usr/home/mmacy/devel/networking/lib/libthr/arch/amd64/amd64/_umtx_op_err.S:37
#1  0x0000000800687571 in __thr_rwlock_rdlock (rwlock=0x800ddc000, flags=<optimized out>, tsp=<optimized out>) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_umtx.c:307
#2  0x000000080068b449 in rwlock_rdlock_common (rwlock=<optimized out>, abstime=0x0) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_rwlock.c:206
#3  0x0000000000211c20 in ztest_dmu_objset_create_destroy (zd=0x800dccbc0, id=7) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:3942
#4  0x000000000022571e in ztest_execute (test=11, zi=0x2271a0 <ztest_info+352>, id=7) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6539
#5  0x0000000000224ff9 in ztest_thread (arg=0x7) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6586
#6  0x0000000800685426 in thread_start (curthread=0x80358e000) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_create.c:291
#7  0x0000000000000000 in ?? ()
Backtrace stopped: Cannot access memory at address 0x7fff94ba5000

Thread 598 (LWP 102922 of process 80572):
#0  _umtx_op_err () at /usr/home/mmacy/devel/networking/lib/libthr/arch/amd64/amd64/_umtx_op_err.S:37
#1  0x0000000800687571 in __thr_rwlock_rdlock (rwlock=0x800ddc000, flags=<optimized out>, tsp=<optimized out>) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_umtx.c:307
#2  0x000000080068b449 in rwlock_rdlock_common (rwlock=<optimized out>, abstime=0x0) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_rwlock.c:206
#3  0x0000000000212e0c in ztest_fault_inject (zd=0x800dd6f40, id=6) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:5679
#4  0x000000000022571e in ztest_execute (test=17, zi=0x227260 <ztest_info+544>, id=6) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6539
#5  0x0000000000224ff9 in ztest_thread (arg=0x6) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6586
#6  0x0000000800685426 in thread_start (curthread=0x810034b00) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_create.c:291
#7  0x0000000000000000 in ?? ()
Backtrace stopped: Cannot access memory at address 0x7fff94da6000

Thread 597 (LWP 101938 of process 80572):
#0  _umtx_op_err () at /usr/home/mmacy/devel/networking/lib/libthr/arch/amd64/amd64/_umtx_op_err.S:37
#1  0x0000000800687571 in __thr_rwlock_rdlock (rwlock=0x800ddc000, flags=<optimized out>, tsp=<optimized out>) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_umtx.c:307
#2  0x000000080068b449 in rwlock_rdlock_common (rwlock=<optimized out>, abstime=0x0) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_rwlock.c:206
#3  0x0000000000212a24 in ztest_spa_create_destroy (zd=0x800dd5400, id=5) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:2746
#4  0x000000000022571e in ztest_execute (test=16, zi=0x227240 <ztest_info+512>, id=5) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6539
#5  0x0000000000224ff9 in ztest_thread (arg=0x5) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6586
#6  0x0000000800685426 in thread_start (curthread=0x807a76e00) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_create.c:291
#7  0x0000000000000000 in ?? ()
Backtrace stopped: Cannot access memory at address 0x7fff94fa7000

Thread 532 (LWP 101653 of process 80572):
#0  _umtx_op_err () at /usr/home/mmacy/devel/networking/lib/libthr/arch/amd64/amd64/_umtx_op_err.S:37
#1  0x0000000800687571 in __thr_rwlock_rdlock (rwlock=0x800ddc000, flags=<optimized out>, tsp=<optimized out>) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_umtx.c:307
#2  0x000000080068b449 in rwlock_rdlock_common (rwlock=<optimized out>, abstime=0x0) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_rwlock.c:206
#3  0x0000000000216e12 in ztest_remap_blocks (zd=0x800dd38c0, id=4) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:5453
#4  0x000000000022571e in ztest_execute (test=30, zi=0x227400 <ztest_info+960>, id=4) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6539
#5  0x0000000000224ff9 in ztest_thread (arg=0x4) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6586
#6  0x0000000800685426 in thread_start (curthread=0x807b5ec00) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_create.c:291
#7  0x0000000000000000 in ?? ()
Backtrace stopped: Cannot access memory at address 0x7fff9d5ea000

Thread 531 (LWP 101642 of process 80572):
#0  _umtx_op_err () at /usr/home/mmacy/devel/networking/lib/libthr/arch/amd64/amd64/_umtx_op_err.S:37
#1  0x0000000800687571 in __thr_rwlock_rdlock (rwlock=0x800ddc000, flags=<optimized out>, tsp=<optimized out>) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_umtx.c:307
#2  0x000000080068b449 in rwlock_rdlock_common (rwlock=<optimized out>, abstime=0x0) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_rwlock.c:206
#3  0x00000000002127c2 in ztest_dmu_snapshot_create_destroy (zd=0x800dd1d80, id=3) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:4043
#4  0x000000000022571e in ztest_execute (test=15, zi=0x227220 <ztest_info+480>, id=3) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6539
#5  0x0000000000224ff9 in ztest_thread (arg=0x3) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6586
#6  0x0000000800685426 in thread_start (curthread=0x807b5e700) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_create.c:291
#7  0x0000000000000000 in ?? ()
Backtrace stopped: Cannot access memory at address 0x7fff9d7eb000

Thread 530 (LWP 101630 of process 80572):
#0  _umtx_op_err () at /usr/home/mmacy/devel/networking/lib/libthr/arch/amd64/amd64/_umtx_op_err.S:37
#1  0x0000000800687571 in __thr_rwlock_rdlock (rwlock=0x800ddc000, flags=<optimized out>, tsp=<optimized out>) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_umtx.c:307
#2  0x000000080068b449 in rwlock_rdlock_common (rwlock=<optimized out>, abstime=0x0) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_rwlock.c:206
#3  0x0000000000212218 in ztest_dsl_prop_get_set (zd=0x800dd0240, id=2) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:5437
#4  0x000000000022571e in ztest_execute (test=12, zi=0x2271c0 <ztest_info+384>, id=2) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6539
#5  0x0000000000224ff9 in ztest_thread (arg=0x2) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6586
Thread 529 (LWP 101609 of process 80572):
#0  _umtx_op_err () at /usr/home/mmacy/devel/networking/lib/libthr/arch/amd64/amd64/_umtx_op_err.S:37
#1  0x0000000800687571 in __thr_rwlock_rdlock (rwlock=0x800ddc000, flags=<optimized out>, tsp=<optimized out>) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_umtx.c:307
#2  0x000000080068b449 in rwlock_rdlock_common (rwlock=<optimized out>, abstime=0x0) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_rwlock.c:206
#3  0x0000000000212218 in ztest_dsl_prop_get_set (zd=0x800dce700, id=1) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:5437
#4  0x000000000022571e in ztest_execute (test=12, zi=0x2271c0 <ztest_info+384>, id=1) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6539
#5  0x0000000000224ff9 in ztest_thread (arg=0x1) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6586
#6  0x0000000800685426 in thread_start (curthread=0x807b10500) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_create.c:291
#7  0x0000000000000000 in ?? ()
Backtrace stopped: Cannot access memory at address 0x7fffa2612000

Thread 528 (LWP 101599 of process 80572):
#0  _umtx_op_err () at /usr/home/mmacy/devel/networking/lib/libthr/arch/amd64/amd64/_umtx_op_err.S:37
#1  0x00000008006873e1 in _thr_umtx_timedwait_uint (mtx=0x807915e30, id=<optimized out>, clockid=<optimized out>, abstime=<optimized out>, shared=<optimized out>) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_umtx.c:236
#2  0x0000000800691038 in cond_wait_user (abstime=0x0, cancel=1, cvp=<optimized out>, mp=<optimized out>) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_cond.c:306
#3  cond_wait_common (cond=<optimized out>, mutex=<optimized out>, abstime=0x0, cancel=1) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_cond.c:366
#4  0x00000008003aeeaa in cv_wait (cv=0x80deb8268, mp=0x80deb8208) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/lib/libzpool/common/kernel.c:329
#5  0x000000080045824d in txg_wait_synced (dp=0x80deb8000, txg=150) at /usr/home/mmacy/devel/networking/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/txg.c:659
#6  0x0000000000213b70 in ztest_ddt_repair (zd=0x800dccbc0, id=0) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:5985
#7  0x000000000022571e in ztest_execute (test=18, zi=0x227280 <ztest_info+576>, id=0) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6539
#8  0x0000000000224ff9 in ztest_thread (arg=0x0) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6586
#9  0x0000000800685426 in thread_start (curthread=0x807b10000) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_create.c:291
#10 0x0000000000000000 in ?? ()
Backtrace stopped: Cannot access memory at address 0x7fffa2813000

Thread 527 (LWP 101576 of process 80572):
#0  _poll () at _poll.S:3
#1  0x0000000800687f46 in __thr_poll (fds=0x0, nfds=0, timeout=360000) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_syscalls.c:338
#2  0x0000000000224cdc in ztest_deadman_thread (arg=0x800dca498) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6498
#3  0x0000000800685426 in thread_start (curthread=0x807b00b00) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_create.c:291
#4  0x0000000000000000 in ?? ()
Backtrace stopped: Cannot access memory at address 0x7fffa2a14000

Thread 526 (LWP 101573 of process 80572):
#0  _poll () at _poll.S:3
#1  0x0000000800687f46 in __thr_poll (fds=0x0, nfds=0, timeout=100) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_syscalls.c:338
#2  0x0000000000224c09 in ztest_resume_thread (arg=0x803589000) at /usr/home/mmacy/devel/networking/cddl/contrib/opensolaris/cmd/ztest/ztest.c:6469
#3  0x0000000800685426 in thread_start (curthread=0x807b00600) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_create.c:291
#4  0x0000000000000000 in ?? ()
Backtrace stopped: Cannot access memory at address 0x7fffa2c15000

Thread 525 (LWP 101559 of process 80572):
#0  0x0000000800560ab5 in refcount_remove_many (rc=0x8006011a0, number=4096, holder=0x831d30080) at /usr/home/mmacy/devel/networking/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/refcount.c:174
#1  0x000000080054b075 in arc_free_data_impl (hdr=0x831d30080, size=4096, tag=0x831d30080) at /usr/home/mmacy/devel/networking/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c:6109
#2  0x000000080054a739 in arc_free_data_abd (hdr=0x831d30080, abd=0x832a2bc80, size=4096, tag=0x831d30080) at /usr/home/mmacy/devel/networking/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c:6074
#3  0x00000008005395c4 in arc_hdr_free_abd (hdr=0x831d30080, free_rdata=B_FALSE) at /usr/home/mmacy/devel/networking/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c:3955
#4  0x000000080053a1d4 in arc_hdr_destroy (hdr=0x831d30080) at /usr/home/mmacy/devel/networking/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c:4479
#5  0x0000000800539ad9 in arc_buf_destroy (buf=0x8324c18c0, tag=0x831d2ff00) at /usr/home/mmacy/devel/networking/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c:4514
#6  0x0000000800523fa3 in dbuf_destroy (db=0x831d2ff00) at /usr/home/mmacy/devel/networking/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:2525
#7  0x0000000800523e86 in dbuf_undirty (db=0x831d2ff00, tx=0x810138a40) at /usr/home/mmacy/devel/networking/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:2252
#8  0x000000080052337c in dbuf_free_range (dn=0x810086000, start_blkid=32792, end_blkid=65559, tx=0x810138a40) at /usr/home/mmacy/devel/networking/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:1715
#9  0x00000008004eeaaf in dnode_free_range (dn=0x810086000, off=134316032, len=134217728, tx=0x810138a40) at /usr/home/mmacy/devel/networking/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dnode.c:1974
#10 0x0000000800517830 in dmu_free_range (os=0x8014ca680, object=176, offset=134316032, size=134217728, tx=0x810138a40) at /usr/home/mmacy/devel/networking/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu.c:939
#11 0x0000000800422e46 in zap_table_grow (zap=0x813e02200, tbl=0x81e6fa010, transfer_func=0x800422890 <zap_ptrtbl_transfer>, tx=0x810138a40) at /usr/home/mmacy/devel/networking/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zap.c:198
#12 0x0000000800422459 in zap_grow_ptrtbl (zap=0x813e02200, tx=0x810138a40) at /usr/home/mmacy/devel/networking/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zap.c:362
#13 0x000000080041f29c in zap_expand_leaf (zn=0x810077680, l=0x810080580, tag=0x8002e5c3a, tx=0x810138a40, lp=0x7fffa3217930) at /usr/home/mmacy/devel/networking/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zap.c:630
#14 0x000000080041fc9a in fzap_update (zn=0x810077680, integer_size=1, num_integers=32, val=0x7fffa3217a40, tag=0x8002e5c3a, tx=0x810138a40) at /usr/home/mmacy/devel/networking/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zap.c:894
#15 0x0000000800417838 in zap_update_uint64 (os=0x8014ca680, zapobj=176, key=0x81e9c3600, key_numints=5, integer_size=1, num_integers=32, val=0x7fffa3217a40, tx=0x810138a40)
    at /usr/home/mmacy/devel/networking/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zap_micro.c:1331
#16 0x000000080051e943 in ddt_zap_update (os=0x8014ca680, object=176, dde=0x81e9c3600, tx=0x810138a40) at /usr/home/mmacy/devel/networking/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/ddt_zap.c:98
#17 0x0000000800557974 in ddt_object_update (ddt=0x80fcc8300, type=DDT_TYPE_ZAP, class=DDT_CLASS_UNIQUE, dde=0x81e9c3600, tx=0x810138a40) at /usr/home/mmacy/devel/networking/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/ddt.c:199
#18 0x000000080055c5b0 in ddt_sync_entry (ddt=0x80fcc8300, dde=0x81e9c3600, tx=0x810138a40, txg=150) at /usr/home/mmacy/devel/networking/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/ddt.c:1085
#19 0x000000080055bb60 in ddt_sync_table (ddt=0x80fcc8300, tx=0x810138a40, txg=150) at /usr/home/mmacy/devel/networking/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/ddt.c:1120
#20 0x000000080055ba0e in ddt_sync (spa=0x803589000, txg=150) at /usr/home/mmacy/devel/networking/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/ddt.c:1172
#21 0x000000080047a2e0 in spa_sync (spa=0x803589000, txg=150) at /usr/home/mmacy/devel/networking/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c:8187
#22 0x0000000800457e54 in txg_sync_thread (arg=0x80deb8000) at /usr/home/mmacy/devel/networking/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/txg.c:543
#23 0x0000000800685426 in thread_start (curthread=0x807aff700) at /usr/home/mmacy/devel/networking/lib/libthr/thread/thr_create.c:291
#24 0x0000000000000000 in ?? ()
Backtrace stopped: Cannot access memory at address 0x7fffa3218000