No pinning (bad boot) [SUM] 0.0- 5.0 sec 12.1 GBytes 20.8 Gbits/sec 11.9 GBytes 20.4 Gbits/sec [SUM] 5.0-10.0 sec 11.9 GBytes 20.4 Gbits/sec 12.4 GBytes 21.3 Gbits/sec [SUM] 10.0-15.0 sec 12.0 GBytes 20.6 Gbits/sec 12.1 GBytes 20.7 Gbits/sec [SUM] 15.0-20.0 sec 12.0 GBytes 20.6 Gbits/sec 12.2 GBytes 21.0 Gbits/sec [SUM] 20.0-25.0 sec 11.9 GBytes 20.5 Gbits/sec 12.5 GBytes 21.4 Gbits/sec [SUM] 25.0-30.0 sec 12.2 GBytes 20.9 Gbits/sec 12.1 GBytes 20.8 Gbits/sec [SUM] 30.0-35.0 sec 12.1 GBytes 20.8 Gbits/sec 12.2 GBytes 21.0 Gbits/sec [SUM] 35.0-40.0 sec 11.8 GBytes 20.3 Gbits/sec 12.2 GBytes 20.9 Gbits/sec [SUM] 40.0-45.0 sec 12.4 GBytes 21.3 Gbits/sec 11.7 GBytes 20.1 Gbits/sec [SUM] 45.0-50.0 sec 12.2 GBytes 21.0 Gbits/sec 12.0 GBytes 20.6 Gbits/sec [SUM] 50.0-55.0 sec 11.9 GBytes 20.5 Gbits/sec 12.0 GBytes 20.7 Gbits/sec [SUM] 55.0-60.0 sec 11.8 GBytes 20.3 Gbits/sec 12.6 GBytes 21.7 Gbits/sec CPU 0: 0.0% user, 0.0% nice, 0.4% system, 77.2% interrupt, 22.4% idle CPU 1: 0.0% user, 0.0% nice, 0.4% system, 0.0% interrupt, 99.6% idle CPU 2: 0.0% user, 0.0% nice, 0.8% system, 9.4% interrupt, 89.8% idle CPU 3: 0.0% user, 0.0% nice, 0.8% system, 9.8% interrupt, 89.4% idle CPU 4: 0.0% user, 0.0% nice, 1.2% system, 7.5% interrupt, 91.3% idle CPU 5: 0.0% user, 0.0% nice, 2.0% system, 9.8% interrupt, 88.2% idle CPU 6: 0.4% user, 0.0% nice, 0.8% system, 7.1% interrupt, 91.7% idle CPU 7: 0.0% user, 0.0% nice, 0.4% system, 11.0% interrupt, 88.6% idle CPU 8: 0.4% user, 0.0% nice, 1.6% system, 9.1% interrupt, 89.0% idle CPU 9: 0.0% user, 0.0% nice, 1.6% system, 13.8% interrupt, 84.6% idle CPU 10: 0.0% user, 0.0% nice, 1.2% system, 8.3% interrupt, 90.6% idle CPU 11: 0.0% user, 0.0% nice, 1.2% system, 11.8% interrupt, 87.0% idle CPU 12: 0.4% user, 0.0% nice, 14.2% system, 0.0% interrupt, 85.4% idle CPU 13: 0.8% user, 0.0% nice, 11.4% system, 0.0% interrupt, 87.8% idle CPU 14: 0.4% user, 0.0% nice, 6.7% system, 0.4% interrupt, 92.5% idle CPU 15: 0.0% user, 0.0% nice, 11.0% system, 0.0% interrupt, 89.0% idle CPU 16: 0.4% user, 0.0% nice, 15.7% system, 0.0% interrupt, 83.9% idle CPU 17: 0.4% user, 0.0% nice, 9.1% system, 0.0% interrupt, 90.6% idle CPU 18: 0.4% user, 0.0% nice, 12.0% system, 0.0% interrupt, 87.6% idle CPU 19: 0.8% user, 0.0% nice, 12.6% system, 0.0% interrupt, 86.6% idle CPU 20: 0.4% user, 0.0% nice, 14.2% system, 0.0% interrupt, 85.4% idle CPU 21: 0.4% user, 0.0% nice, 13.4% system, 0.0% interrupt, 86.2% idle CPU 22: 0.8% user, 0.0% nice, 9.4% system, 0.0% interrupt, 89.8% idle CPU 23: 0.4% user, 0.0% nice, 15.1% system, 0.0% interrupt, 84.5% idle |---------------------------------------||---------------------------------------| |-- Socket 0 --||-- Socket 1 --| |---------------------------------------||---------------------------------------| |-- Memory Channel Monitoring --||-- Memory Channel Monitoring --| |---------------------------------------||---------------------------------------| |-- Mem Ch 0: Reads (MB/s): 464.67 --||-- Mem Ch 0: Reads (MB/s): 254.42 --| |-- Writes(MB/s): 726.32 --||-- Writes(MB/s): 245.35 --| |-- Mem Ch 1: Reads (MB/s): 493.93 --||-- Mem Ch 1: Reads (MB/s): 250.55 --| |-- Writes(MB/s): 732.61 --||-- Writes(MB/s): 240.89 --| |-- Mem Ch 2: Reads (MB/s): 455.33 --||-- Mem Ch 2: Reads (MB/s): 248.38 --| |-- Writes(MB/s): 722.65 --||-- Writes(MB/s): 241.55 --| |-- Mem Ch 3: Reads (MB/s): 471.34 --||-- Mem Ch 3: Reads (MB/s): 250.86 --| |-- Writes(MB/s): 722.21 --||-- Writes(MB/s): 242.23 --| |-- NODE 0 Mem Read (MB/s) : 1885.26 --||-- NODE 1 Mem Read (MB/s) : 1004.21 --| |-- NODE 0 Mem Write(MB/s) : 2903.79 --||-- NODE 1 Mem Write(MB/s) : 970.02 --| |-- NODE 0 P. Write (T/s): 61576 --||-- NODE 1 P. Write (T/s): 61383 --| |-- NODE 0 Memory (MB/s): 4789.05 --||-- NODE 1 Memory (MB/s): 1974.23 --| |---------------------------------------||---------------------------------------| |---------------------------------------||---------------------------------------| |-- System Read Throughput(MB/s): 2889.47 --| |-- System Write Throughput(MB/s): 3873.81 --| |-- System Memory Throughput(MB/s): 6763.28 --| |---------------------------------------||---------------------------------------| Time elapsed: 1001 ms Called sleep function for 1000 ms |---------------------------------------||---------------------------------------| |-- Socket 0 --||-- Socket 1 --| |---------------------------------------||---------------------------------------| |-- Memory Channel Monitoring --||-- Memory Channel Monitoring --| |---------------------------------------||---------------------------------------| |-- Mem Ch 0: Reads (MB/s): 433.82 --||-- Mem Ch 0: Reads (MB/s): 236.55 --| |-- Writes(MB/s): 694.92 --||-- Writes(MB/s): 219.55 --| |-- Mem Ch 1: Reads (MB/s): 452.07 --||-- Mem Ch 1: Reads (MB/s): 223.48 --| |-- Writes(MB/s): 700.23 --||-- Writes(MB/s): 214.13 --| |-- Mem Ch 2: Reads (MB/s): 416.95 --||-- Mem Ch 2: Reads (MB/s): 215.39 --| |-- Writes(MB/s): 688.50 --||-- Writes(MB/s): 212.55 --| |-- Mem Ch 3: Reads (MB/s): 450.80 --||-- Mem Ch 3: Reads (MB/s): 226.96 --| |-- Writes(MB/s): 694.09 --||-- Writes(MB/s): 215.19 --| |-- NODE 0 Mem Read (MB/s) : 1753.64 --||-- NODE 1 Mem Read (MB/s) : 902.38 --| |-- NODE 0 Mem Write(MB/s) : 2777.74 --||-- NODE 1 Mem Write(MB/s) : 861.42 --| |-- NODE 0 P. Write (T/s): 61563 --||-- NODE 1 P. Write (T/s): 61335 --| |-- NODE 0 Memory (MB/s): 4531.38 --||-- NODE 1 Memory (MB/s): 1763.80 --| |---------------------------------------||---------------------------------------| |---------------------------------------||---------------------------------------| |-- System Read Throughput(MB/s): 2656.01 --| |-- System Write Throughput(MB/s): 3639.16 --| |-- System Memory Throughput(MB/s): 6295.17 --| |---------------------------------------||---------------------------------------| Time elapsed: 1010 ms Called sleep function for 1000 ms |---------------------------------------||---------------------------------------| |-- Socket 0 --||-- Socket 1 --| |---------------------------------------||---------------------------------------| |-- Memory Channel Monitoring --||-- Memory Channel Monitoring --| |---------------------------------------||---------------------------------------| |-- Mem Ch 0: Reads (MB/s): 544.32 --||-- Mem Ch 0: Reads (MB/s): 234.10 --| |-- Writes(MB/s): 762.36 --||-- Writes(MB/s): 234.81 --| |-- Mem Ch 1: Reads (MB/s): 572.30 --||-- Mem Ch 1: Reads (MB/s): 226.99 --| |-- Writes(MB/s): 769.77 --||-- Writes(MB/s): 231.25 --| |-- Mem Ch 2: Reads (MB/s): 527.05 --||-- Mem Ch 2: Reads (MB/s): 226.94 --| |-- Writes(MB/s): 756.64 --||-- Writes(MB/s): 230.84 --| |-- Mem Ch 3: Reads (MB/s): 541.23 --||-- Mem Ch 3: Reads (MB/s): 226.83 --| |-- Writes(MB/s): 757.24 --||-- Writes(MB/s): 231.53 --| |-- NODE 0 Mem Read (MB/s) : 2184.89 --||-- NODE 1 Mem Read (MB/s) : 914.87 --| |-- NODE 0 Mem Write(MB/s) : 3046.02 --||-- NODE 1 Mem Write(MB/s) : 928.42 --| |-- NODE 0 P. Write (T/s): 61974 --||-- NODE 1 P. Write (T/s): 61377 --| |-- NODE 0 Memory (MB/s): 5230.91 --||-- NODE 1 Memory (MB/s): 1843.29 --| |---------------------------------------||---------------------------------------| |---------------------------------------||---------------------------------------| |-- System Read Throughput(MB/s): 3099.76 --| |-- System Write Throughput(MB/s): 3974.44 --| |-- System Memory Throughput(MB/s): 7074.20 --| |---------------------------------------||---------------------------------------| Time elapsed: 1002 ms Called sleep function for 1000 ms |---------------------------------------||---------------------------------------| |-- Socket 0 --||-- Socket 1 --| |---------------------------------------||---------------------------------------| |-- Memory Channel Monitoring --||-- Memory Channel Monitoring --| |---------------------------------------||---------------------------------------| |-- Mem Ch 0: Reads (MB/s): 474.46 --||-- Mem Ch 0: Reads (MB/s): 240.85 --| |-- Writes(MB/s): 743.64 --||-- Writes(MB/s): 234.09 --| |-- Mem Ch 1: Reads (MB/s): 505.26 --||-- Mem Ch 1: Reads (MB/s): 242.64 --| |-- Writes(MB/s): 750.09 --||-- Writes(MB/s): 231.83 --| |-- Mem Ch 2: Reads (MB/s): 464.54 --||-- Mem Ch 2: Reads (MB/s): 236.20 --| |-- Writes(MB/s): 739.12 --||-- Writes(MB/s): 230.58 --| |-- Mem Ch 3: Reads (MB/s): 500.48 --||-- Mem Ch 3: Reads (MB/s): 240.84 --| |-- Writes(MB/s): 744.36 --||-- Writes(MB/s): 231.99 --| |-- NODE 0 Mem Read (MB/s) : 1944.74 --||-- NODE 1 Mem Read (MB/s) : 960.53 --| |-- NODE 0 Mem Write(MB/s) : 2977.21 --||-- NODE 1 Mem Write(MB/s) : 928.49 --| |-- NODE 0 P. Write (T/s): 61646 --||-- NODE 1 P. Write (T/s): 61386 --| |-- NODE 0 Memory (MB/s): 4921.95 --||-- NODE 1 Memory (MB/s): 1889.02 --| |---------------------------------------||---------------------------------------| |---------------------------------------||---------------------------------------| |-- System Read Throughput(MB/s): 2905.27 --| |-- System Write Throughput(MB/s): 3905.70 --| |-- System Memory Throughput(MB/s): 6810.97 --| |---------------------------------------||---------------------------------------| Time elapsed: 1000 ms Called sleep function for 1000 ms |---------------------------------------||---------------------------------------| |-- Socket 0 --||-- Socket 1 --| |---------------------------------------||---------------------------------------| |-- Memory Channel Monitoring --||-- Memory Channel Monitoring --| |---------------------------------------||---------------------------------------| |-- Mem Ch 0: Reads (MB/s): 472.34 --||-- Mem Ch 0: Reads (MB/s): 225.99 --| |-- Writes(MB/s): 749.36 --||-- Writes(MB/s): 224.26 --| |-- Mem Ch 1: Reads (MB/s): 503.32 --||-- Mem Ch 1: Reads (MB/s): 222.55 --| |-- Writes(MB/s): 756.38 --||-- Writes(MB/s): 220.32 --| |-- Mem Ch 2: Reads (MB/s): 469.76 --||-- Mem Ch 2: Reads (MB/s): 228.01 --| |-- Writes(MB/s): 748.84 --||-- Writes(MB/s): 221.72 --| |-- Mem Ch 3: Reads (MB/s): 499.20 --||-- Mem Ch 3: Reads (MB/s): 227.93 --| |-- Writes(MB/s): 749.92 --||-- Writes(MB/s): 222.87 --| |-- NODE 0 Mem Read (MB/s) : 1944.62 --||-- NODE 1 Mem Read (MB/s) : 904.49 --| |-- NODE 0 Mem Write(MB/s) : 3004.50 --||-- NODE 1 Mem Write(MB/s) : 889.18 --| |-- NODE 0 P. Write (T/s): 61719 --||-- NODE 1 P. Write (T/s): 61337 --| |-- NODE 0 Memory (MB/s): 4949.12 --||-- NODE 1 Memory (MB/s): 1793.66 --| |---------------------------------------||---------------------------------------| |---------------------------------------||---------------------------------------| |-- System Read Throughput(MB/s): 2849.11 --| |-- System Write Throughput(MB/s): 3893.67 --| |-- System Memory Throughput(MB/s): 6742.78 --| |---------------------------------------||---------------------------------------| srv1 max wait_max total wait_total count avg wait_avg cnt_hold cnt_lock name 14 15 2449267 75056 4249271 0 0 0 47796 /usr/src/sys/netinet/tcp_output.c:328 (sleep mutex:so_snd) 2 5 517367 49726 2969213 0 0 0 205820 /usr/src/sys/kern/kern_timeout.c:578 (spin mutex:callout) 11 11 645096 43575 2945064 0 0 0 29713 /usr/src/sys/kern/uipc_socket.c:1756 (sleep mutex:so_rcv) 13 8 2467960 37666 2602703 0 0 0 47507 /usr/src/sys/kern/subr_sleepqueue.c:254 (spin mutex:sleepq chain) 3 4 103860 6060 1851968 0 0 0 36791 /usr/src/sys/dev/random/random_harvestq.c:419 (spin mutex:entropy harvest mutex) 87 35 10802976 139094 1850347 5 0 0 67421 /usr/src/sys/netinet/in_pcb.c:1964 (rw:tcpinp) 8 0 459431 0 1850347 0 0 0 0 /usr/src/sys/netinet/in_pcb.c:1957 (rw:pcbinfohash) 8 11 372723 27127 1815174 0 0 0 19205 /usr/src/sys/kern/uipc_socket.c:1529 (sleep mutex:so_rcv) 14 0 970118 0 1714763 0 0 0 0 /usr/src/sys/modules/mlxen/../../ofed/drivers/net/mlx4/en_tx.c:1084 (sleep mutex:mlx4 tx) 11 0 219352 0 1683702 0 0 0 0 /usr/src/sys/modules/mlx4/../../ofed/drivers/net/mlx4/cq.c:59 (rw:lnxrw) 0 0 101740 0 1384252 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 0) 13 23 193246 15784 1238666 0 0 0 5301 /usr/src/sys/kern/uipc_socket.c:1258 (sleep mutex:so_snd) 14 20 1294303 94865 1015502 1 0 0 50729 /usr/src/sys/netinet/tcp_input.c:1908 (sleep mutex:so_rcv) 20 57 492019 250795 951468 0 0 0 59769 /usr/src/sys/netinet/tcp_usrreq.c:814 (rw:tcpinp) 4 0 47648 0 951468 0 0 0 0 /usr/src/sys/kern/uipc_socket.c:1919 (sleep mutex:so_rcv) 489134 0 156952451 0 951457 164 0 0 0 /usr/src/sys/kern/uipc_sockbuf.c:283 (sx:so_rcv_sx) 15 14 60808 44945 863706 0 0 0 22613 /usr/src/sys/kern/kern_mutex.c:190 (sleep mutex:so_rcv) 44 220 696724 2279500 813241 0 2 0 248023 /usr/src/sys/netinet/tcp_usrreq.c:874 (rw:tcpinp) 10 0 231352 0 813241 0 0 0 0 /usr/src/sys/kern/uipc_sockbuf.c:682 (sleep mutex:so_snd) 1 3 178215 73 699382 0 0 0 117 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 0) 1 6 178163 29 692643 0 0 0 21 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 0) 0 3 60828 13 691784 0 0 0 12 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 0) 32 5 597781 1223 516635 1 0 0 964 /usr/src/sys/netinet/tcp_input.c:2826 (sleep mutex:so_snd) 9 43 32162 1115782 425425 0 2 0 336225 /usr/src/sys/kern/kern_mutex.c:190 (sleep mutex:so_snd) 3 0 60820 0 252001 0 0 0 0 /usr/src/sys/kern/kern_clocksource.c:216 (spin mutex:et_hw_mtx) 0 0 23880 0 246700 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 10) 0 0 23902 0 237870 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 11) 0 0 23610 0 237856 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 8) 2 0 16742 0 229239 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 13) 0 0 22801 0 228333 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 9) 0 0 22454 0 227791 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 6) 0 0 21870 0 222103 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 7) 0 0 21833 0 220520 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 4) 0 0 21453 0 215044 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 5) 0 0 21040 0 213619 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 2) 3 0 13892 0 206500 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 20) 1 0 13736 0 205363 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 16) 0 0 20140 0 203977 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 3) 2 0 13091 0 200936 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 15) 3 0 13523 0 200658 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 21) 718027 0 156823927 0 200336 782 0 0 0 /usr/src/sys/kern/uipc_sockbuf.c:283 (sx:so_snd_sx) 2 0 13173 0 198624 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 18) 2 0 12888 0 195283 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 22) 1 4 18633 52 193821 0 0 0 104 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 0) 9 6 166744 142 189337 0 0 0 116 /usr/src/sys/netinet/tcp_input.c:1808 (sleep mutex:so_snd) 7 9 74969 649 189337 0 0 0 449 /usr/src/sys/kern/uipc_sockbuf.c:1148 (sleep mutex:so_snd) 3 0 12183 0 186172 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 17) 2 0 12218 0 184949 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 14) 2 0 12239 0 177281 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 23) 2 0 11529 0 159920 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 12) 2 0 10532 0 155085 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 19) 1 1 20827 319 122478 0 0 0 706 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 10) 1 1 24724 312 118109 0 0 0 759 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 11) 0 3 20128 286 117994 0 0 0 685 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 8) 1 3 32178 16 114578 0 0 0 19 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 13) 3 0 15720 0 114525 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 13) 1 0 6394 0 114525 0 0 0 1 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 13) 1 0 6993 0 114321 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 13) 0 1 20380 263 113331 0 0 0 775 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 9) 0 2 19575 333 113017 0 0 0 858 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 6) 3 1 29223 38 111946 0 0 0 122 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 10) 0 1 19190 261 110259 0 0 0 740 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 7) 4 2 61482 492 109906 0 0 0 1144 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 13) 0 1 18267 302 109314 0 0 0 778 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 4) 1 3 55564 1951 108696 0 0 0 7281 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 10) 0 1 34636 55 108695 0 0 0 138 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 10) 3 1 28509 52 108460 0 0 0 173 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 8) 4 2 28243 44 108407 0 0 0 131 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 11) 70 0 870696 0 107586 8 0 0 0 /usr/src/sys/modules/mlxen/../../ofed/drivers/net/mlx4/en_tx.c:511 (sleep mutex:mlx4 comp) 0 2 19446 257 106678 0 0 0 685 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 5) 0 2 18167 286 105895 0 0 0 651 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 2) 1 1 53652 1974 104537 0 0 0 7429 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 11) 0 0 33568 58 104534 0 0 0 149 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 11) 2 1 27227 53 104481 0 0 0 184 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 6) 3 0 27363 35 104459 0 0 0 131 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 9) 1 3 53351 1758 103946 0 0 0 6995 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 8) 0 2 32762 58 103945 0 0 0 128 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 8) 2 3 28160 26 103221 0 0 0 24 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 20) 3 0 14574 0 103152 0 0 0 1 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 20) 3 0 5671 0 103152 0 0 0 1 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 20) 1 0 6426 0 102902 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 20) 4 2 29332 16 102636 0 0 0 22 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 16) 3 0 5765 0 102581 0 0 0 1 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 16) 2 0 14280 0 102581 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 16) 2 1 26520 49 102251 0 0 0 164 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 7) 1 0 6256 0 101988 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 16) 2 2 27236 68 101607 0 0 0 204 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 4) 0 2 17222 222 101119 0 0 0 506 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 3) 2 2 28619 21 100449 0 0 0 25 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 15) 1 2 51558 1723 100380 0 0 0 6868 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 9) 0 0 32017 35 100379 0 0 0 95 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 9) 2 1 13446 1 100366 0 0 0 1 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 15) 1 0 5651 0 100366 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 15) 2 3 28140 30 100289 0 0 0 24 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 21) 3 0 13797 0 100227 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 21) 2 1 5382 1 100227 0 0 0 1 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 21) 0 0 6351 0 99991 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 15) 1 0 6358 0 99923 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 21) 5 3 25991 56 99383 0 0 0 187 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 5) 1 2 51040 1795 99217 0 0 0 6942 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 6) 0 1 31511 51 99215 0 0 0 128 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 6) 3 2 27695 25 99089 0 0 0 26 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 18) 2 1 13195 1 98942 0 0 0 1 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 18) 1 0 5438 0 98942 0 0 0 1 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 18) 2 1 25720 55 98839 0 0 0 199 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 2) 2 0 6116 0 98302 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 18) 5 3 54873 368 98163 0 0 0 822 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 20) 5 3 56931 514 97987 0 0 0 1427 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 16) 3 2 26474 26 97610 0 0 0 22 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 22) 2 1 13309 2 97534 0 0 0 3 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 22) 1 0 5418 0 97534 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 22) 2 0 6180 0 97348 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 22) 9 0 22331 0 97280 0 0 0 0 /usr/src/sys/dev/random/fortuna.c:222 (sleep mutex:reseed mutex) 6 2 55908 603 96443 0 0 0 1511 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 15) 2 2 49437 1616 96190 0 0 0 6324 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 7) 0 1 30282 40 96187 0 0 0 101 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 7) 4 2 56459 359 95825 0 0 0 798 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 21) 4 9 56095 442 94809 0 0 0 1015 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 18) 1 1 48957 1799 94789 0 0 0 6739 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 4) 0 0 30681 43 94784 0 0 0 109 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 4) 5 1 24706 52 94592 0 0 0 178 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 3) 7 4 55339 342 93401 0 0 0 760 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 22) 3 3 26064 33 93049 0 0 0 29 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 17) 2 0 12849 0 93011 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 17) 1 1 5176 1 93011 0 0 0 1 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 17) 2 0 5716 0 92838 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 17) 2 2 25405 27 92432 0 0 0 26 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 14) 3 0 12490 0 92383 0 0 0 1 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 14) 1 4 5089 4 92383 0 0 0 1 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 14) 1 0 5664 0 92173 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 14) 1 4 46992 1593 91404 0 0 0 6495 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 5) 0 1 28969 40 91399 0 0 0 101 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 5) 1 2 46484 1537 91148 0 0 0 6025 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 2) 0 2 28872 48 91145 0 0 0 112 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 2) 2 3 25034 32 88608 0 0 0 24 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 23) 2 0 12426 0 88522 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 23) 1 0 4984 0 88522 0 0 0 2 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 23) 4 2 54468 372 88517 0 0 0 822 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 17) 4 2 49435 363 88213 0 0 0 1135 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 14) 1 0 5694 0 87854 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 23) 1 4 44978 1476 87406 0 0 0 5795 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 3) 1 1 28039 39 87401 0 0 0 93 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 3) 5 2 47676 301 83888 0 0 0 720 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 23) 2 8 22616 48 79925 0 0 0 54 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 12) 2 0 11058 0 79864 0 0 0 1 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 12) 1 0 4497 0 79864 0 0 0 1 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 12) 2 0 4976 0 79478 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 12) 4 3 21484 22 77518 0 0 0 17 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 19) 3 0 10838 0 77434 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 19) 2 0 4372 0 77434 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 19) 2 0 4884 0 77200 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 19) 6 2 43846 282 75120 0 0 0 637 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 12) 5 4 40986 298 73218 0 0 0 633 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 19) 2 0 11021 0 63300 0 0 0 0 /usr/src/sys/kern/kern_clocksource.c:760 (spin mutex:et_hw_mtx) 4 0 3985 0 32495 0 0 0 0 /usr/src/sys/vm/uma_core.c:2724 (sleep mutex:mbuf) 7 0 5922 0 32484 0 0 0 0 /usr/src/sys/vm/uma_core.c:2206 (sleep mutex:mbuf) 29 16 44882 1751 24263 1 0 0 870 /usr/src/sys/netinet/tcp_reass.c:316 (sleep mutex:so_rcv) 2 0 4006 0 23639 0 0 0 0 /usr/src/sys/vm/uma_core.c:2206 (sleep mutex:mbuf_jumbo_page) 4 0 3127 0 23637 0 0 0 0 /usr/src/sys/vm/uma_core.c:2724 (sleep mutex:mbuf_jumbo_page) 0 0 2188 10 16124 0 0 0 16 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 5) 0 0 917 6 16122 0 0 0 15 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 5) 0 0 1082 0 15762 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 5) 0 1 2100 7 15665 0 0 0 10 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 2) 0 0 903 7 15665 0 0 0 17 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 2) 0 1 2120 7 15479 0 0 0 9 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 4) 0 0 992 6 15479 0 0 0 16 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 4) 0 0 1033 0 15348 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 2) 0 0 1030 0 15226 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 4) 0 0 2009 6 14981 0 0 0 9 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 8) 0 1 949 11 14979 0 0 0 22 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 8) 0 1 2010 4 14864 0 0 0 5 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 7) 0 0 904 5 14863 0 0 0 14 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 7) 0 1 1971 7 14682 0 0 0 10 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 6) 0 2 879 13 14679 0 0 0 23 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 6) 0 0 1988 4 14655 0 0 0 7 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 10) 0 1 864 14 14654 0 0 0 27 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 10) 0 0 1026 0 14621 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 8) 0 0 1014 0 14599 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 7) 0 0 1946 4 14588 0 0 0 7 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 3) 0 0 859 6 14588 0 0 0 15 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 3) 0 0 1949 7 14403 0 0 0 11 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 11) 0 0 865 3 14402 0 0 0 8 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 11) 0 0 1051 0 14354 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 6) 0 0 993 0 14277 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 10) 0 0 976 0 14252 0 0 0 1 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 3) 0 0 1001 0 14132 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 11) 0 0 1870 7 13788 0 0 0 12 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 9) 0 0 965 6 13787 0 0 0 18 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 9) 0 0 949 0 13516 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 9) 3 3 1154 85 11626 0 0 0 59 /usr/src/sys/netinet/tcp_input.c:2678 (sleep mutex:so_snd) 4 0 1888 0 10540 0 0 0 0 /usr/src/sys/vm/uma_core.c:2206 (sleep mutex:mbuf_jumbo_9k) 2 0 1371 0 10538 0 0 0 0 /usr/src/sys/vm/uma_core.c:2724 (sleep mutex:mbuf_jumbo_9k) 11 8 5028 429 6733 0 0 0 189 /usr/src/sys/modules/mlxen/../../ofed/drivers/net/mlx4/en_tx.c:1055 (sleep mutex:mlx4 tx) 0 2 550 7 6316 0 0 0 8 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sleepq chain) 0 2 525 7 6225 0 0 0 5 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sleepq chain) 1 0 1017 0 5996 0 0 0 0 /usr/src/sys/kern/kern_clocksource.c:827 (spin mutex:et_hw_mtx) 0 1 740 24 5628 0 0 0 34 /usr/src/sys/kern/subr_taskqueue.c:284 (spin mutex:mlx4_en_que) 3 0 2455 7 5605 0 0 0 28 /usr/src/sys/kern/subr_taskqueue.c:467 (spin mutex:mlx4_en_que) 4 0 1421 0 5422 0 0 0 0 /usr/src/sys/kern/kern_synch.c:309 (spin mutex:mlx4_en_que) 3 1 3769 2 5032 0 0 0 3 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 20) 1 0 979 1 4992 0 0 0 4 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 20) 7 0 3673 0 4786 0 0 0 0 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 12) 3 6 984 10 4753 0 0 0 3 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 12) 3 1 3426 2 4704 0 0 0 2 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 23) 3 0 3417 0 4651 0 0 0 0 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 13) 2 1 919 1 4638 0 0 0 2 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 23) 4 1 3497 1 4634 0 0 0 1 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 16) 2 0 918 0 4617 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 13) 2 0 905 1 4605 0 0 0 2 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 16) 3 1 3335 4 4509 0 0 0 3 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 17) 2 1 952 2 4467 0 0 0 3 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 17) 3 1 3097 1 4451 0 0 0 1 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 18) 5 0 3298 0 4438 0 0 0 1 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 21) 2 0 827 2 4375 0 0 0 6 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 21) 4 0 3138 0 4279 0 0 0 1 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 19) 1 3 853 5 4221 0 0 0 3 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 19) 4 0 3090 0 4199 0 0 0 0 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 14) 7 0 3109 0 4186 0 0 0 0 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 22) 2 1 828 2 4157 0 0 0 3 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 14) 2 0 825 0 4131 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 22) 5 2 3125 54 4130 0 0 0 176 /usr/src/sys/kern/kern_timeout.c:751 (spin mutex:callout) 2 2 823 4 4124 0 0 0 4 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 18) 7 0 3444 8 4093 0 0 0 25 /usr/src/sys/kern/kern_timeout.c:449 (spin mutex:callout) 6 2 2991 2 3983 0 0 0 1 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 15) 1 2 757 6 3922 0 0 0 4 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 15) 5 0 749 0 3171 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:560 (spin mutex:turnstile lock) 3 0 1537 0 3171 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:551 (spin mutex:turnstile chain) 9 1 4945 11 2113 2 0 0 14 /usr/src/sys/kern/subr_sleepqueue.c:921 (spin mutex:sleepq chain) 0 0 299 0 2018 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 1) 1 1 584 37 1580 0 0 0 124 /usr/src/sys/kern/kern_timeout.c:864 (spin mutex:callout) 6 6 1075 753 1394 0 0 0 753 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 0) 1 0 95 0 1280 0 0 0 0 /usr/src/sys/modules/mlxen/../../ofed/drivers/net/mlx4/en_netdev.c:1097 (sleep mutex:lnxspin) 1 0 61 0 1280 0 0 0 0 /usr/src/sys/netinet/ip_reass.c:491 (sleep mutex:IP reassembly) 0 0 162 0 1272 0 0 0 0 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 0) 0 0 238 2 1271 0 0 0 7 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 10) 0 0 234 6 1270 0 0 0 16 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 11) 0 0 232 3 1269 0 0 0 9 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 7) 0 0 224 2 1269 0 0 0 7 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 6) 0 0 241 2 1268 0 0 0 7 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 2) 0 0 239 2 1268 0 0 0 7 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 3) 0 0 231 2 1268 0 0 0 8 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 4) 0 0 230 1 1268 0 0 0 6 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 8) 0 0 224 1 1267 0 0 0 7 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 5) 0 0 228 5 1266 0 0 0 13 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 9) 1 0 332 0 1228 0 0 0 0 /usr/src/sys/vm/uma_core.c:2206 (sleep mutex:256 Bucket) 1 0 204 0 1225 0 0 0 0 /usr/src/sys/vm/uma_core.c:2724 (sleep mutex:256 Bucket) 4 0 1074 0 1139 0 0 0 1 /usr/src/sys/kern/kern_timeout.c:364 (spin mutex:callout) 4 0 434 0 1086 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 1) 0 1 127 28 1079 0 0 0 57 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 1) 0 1 320 15 1023 0 0 0 25 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 1) 2 0 489 2 985 0 0 0 4 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 18) 3 0 498 1 980 0 0 0 2 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 22) 3 0 441 0 966 0 0 0 1 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 13) 2 2 422 4 939 0 0 0 5 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 21) 2 1 416 3 932 0 0 0 3 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 15) 2 1 458 3 917 0 0 0 4 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 14) 4 0 430 0 893 0 0 0 0 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 16) 3 0 404 0 890 0 0 0 2 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 20) 3 2 434 2 889 0 0 0 2 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 19) 3 0 419 0 883 0 0 0 1 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 23) 2 0 387 0 873 0 0 0 1 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 17) 0 0 168 1 820 0 0 0 3 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 4) 0 0 167 0 812 0 0 0 2 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 8) 4 2 459 2 796 0 0 0 2 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 12) 3 0 176 0 778 0 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:5515 (rw:pmap pv list) 0 0 165 0 776 0 0 0 0 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 2) 0 0 163 0 774 0 0 0 1 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 3) 0 3 65 4 768 0 0 0 3 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 4) 5 0 182 0 745 0 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:3386 (rw:pmap pv list) 0 0 59 1 735 0 0 0 4 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 3) 44 0 3792 0 726 5 0 0 0 /usr/src/sys/netinet/tcp_timer.c:284 (rw:tcpinp) 0 0 159 0 726 0 0 0 0 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 10) 0 0 57 1 724 0 0 0 3 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 2) 0 0 152 0 722 0 0 0 0 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 6) 0 0 152 0 697 0 0 0 0 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 5) 0 0 143 0 695 0 0 0 0 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 7) 0 0 154 0 692 0 0 0 1 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 9) 0 1 57 3 689 0 0 0 5 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 10) 0 0 168 0 684 0 0 0 1 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 11) 0 0 55 1 684 0 0 0 3 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 6) 232 2 4441 2 682 6 0 0 1 /usr/src/sys/kern/kern_proc.c:1450 (sleep mutex:process lock) 103 0 1695 0 682 2 0 0 0 /usr/src/sys/kern/kern_proc.c:957 (spin mutex:pstatl) 0 0 62 0 682 0 0 0 0 /usr/src/sys/kern/kern_proc.c:924 (sleep mutex:sigacts) 0 0 35 0 682 0 0 0 0 /usr/src/sys/kern/kern_proc.c:978 (sleep mutex:session) 0 0 52 0 676 0 0 0 1 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 8) 1 0 72 0 674 0 0 0 0 /usr/src/sys/kern/kern_proc.c:301 (sleep mutex:process lock) 0 0 75 0 672 0 0 0 0 /usr/src/sys/kern/kern_proc.c:323 (sx:allproc) 0 0 196 0 666 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 1) 0 0 97 0 666 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 1) 0 0 54 2 666 0 0 0 4 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 7) 4 0 178 0 660 0 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:5984 (sleep mutex:pmap) 0 0 50 1 656 0 0 0 2 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 5) 0 0 120 0 643 0 0 0 0 /usr/src/sys/kern/kern_proc.c:1030 (spin mutex:sleepq chain) 0 0 49 2 639 0 0 0 5 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 9) 1 0 49 0 635 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 23) 0 0 113 0 629 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 1) 0 0 49 0 625 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 18) 0 0 53 1 622 0 0 0 3 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 11) 1 0 51 0 569 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 16) 3 0 378 1 561 0 0 0 3 /usr/src/sys/kern/subr_taskqueue.c:467 (spin mutex:mlx4_en_que) 0 1 84 3 561 0 0 0 4 /usr/src/sys/kern/subr_taskqueue.c:284 (spin mutex:mlx4_en_que) 0 2 80 7 556 0 0 0 5 /usr/src/sys/kern/subr_taskqueue.c:284 (spin mutex:mlx4_en_que) 4 0 303 0 554 0 0 0 2 /usr/src/sys/kern/subr_taskqueue.c:467 (spin mutex:mlx4_en_que) 2 0 179 0 549 0 0 0 0 /usr/src/sys/kern/kern_synch.c:309 (spin mutex:mlx4_en_que) 1 0 148 0 548 0 0 0 0 /usr/src/sys/kern/kern_synch.c:309 (spin mutex:mlx4_en_que) 72 0 4100 0 534 7 0 0 0 /usr/src/sys/modules/mlxen/../../ofed/drivers/net/mlx4/en_tx.c:480 (sleep mutex:mlx4 comp) 0 1 126 6 509 0 0 0 22 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 0) 0 0 33 6 443 0 0 0 13 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 0) 0 0 35 4 437 0 0 0 9 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 0) 9 0 85 0 407 0 0 0 0 /usr/src/sys/vm/vm_object.c:529 (rw:vm object) 0 0 96 8 396 0 0 0 15 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 1) 6 2 392 4 366 1 0 0 5 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 18) 4 1 210 3 366 0 0 0 4 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 18) 0 0 33 0 363 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 10) 0 0 32 0 360 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 12) 0 0 18 0 354 0 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:5694 (sleep mutex:pmap) 0 0 22 0 352 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 15) 7 0 409 0 351 1 0 0 0 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 0) 0 0 29 0 351 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 5) 1 1 199 6 343 0 0 0 20 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 1) 0 0 145 0 343 0 0 0 0 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 1) 2 0 44 0 336 0 0 0 0 /usr/src/sys/vm/vm_glue.c:171 (sx:vm map (user)) 0 0 29 0 326 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 3) 14 0 665 0 320 2 0 0 0 /usr/src/sys/modules/mlx4/../../ofed/drivers/net/mlx4/cmd.c:479 (sx:lnxmtx) 4 0 53 0 320 0 0 0 0 /usr/src/sys/kern/kern_sema.c:79 (sleep mutex:lnxsema) 1 0 67 0 320 0 0 0 0 /usr/src/sys/kern/kern_sema.c:94 (sleep mutex:lnxsema) 1 0 61 0 320 0 0 0 0 /usr/src/sys/modules/mlx4/../../ofed/drivers/net/mlx4/cmd.c:780 (sleep mutex:lnxspin) 1 0 54 0 320 0 0 0 0 /usr/src/sys/modules/mlx4/../../ofed/drivers/net/mlx4/cmd.c:719 (sleep mutex:lnxspin) 0 0 29 0 316 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 8) 0 0 29 0 313 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 6) 0 0 26 0 307 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 2) 74 0 2734 0 302 9 0 0 0 /usr/src/sys/netinet/tcp_timer.c:587 (rw:tcpinp) 7 0 327 0 302 1 0 0 0 /usr/src/sys/netinet/tcp_timer.c:584 (rw:tcp) 17 0 851 0 300 2 0 0 0 /usr/src/sys/vm/vm_map.c:3985 (sx:vm map (user)) 5 0 96 0 300 0 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:4367 (sleep mutex:pmap) 3 0 87 0 288 0 0 0 0 /usr/src/sys/vm/vm_fault.c:1010 (rw:vm object) 1 0 46 0 288 0 0 0 0 /usr/src/sys/vm/vm_fault.c:1011 (sleep mutex:vm page) 0 0 21 0 288 0 0 0 0 /usr/src/sys/vm/vm_page.c:2796 (sleep mutex:vm active pagequeue) 0 0 19 0 288 0 0 0 2 /usr/src/sys/vm/vm_page.c:2840 (sleep mutex:vm active pagequeue) 0 0 27 0 286 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 21) 0 0 25 0 255 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 11) 2 1 53 2 252 0 0 0 3 /usr/src/sys/vm/vm_page.c:1651 (sleep mutex:vm page free queue) 1 0 53 0 248 0 0 0 0 /usr/src/sys/dev/usb/usb_hub.c:2371 (sleep mutex:ehci1) 1 0 27 0 248 0 0 0 0 /usr/src/sys/vm/vm_page.c:3037 (sleep mutex:vm page free queue) 0 0 22 0 248 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 9) 0 0 23 0 245 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 4) 0 0 25 0 239 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 7) 2 0 60 0 230 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:5302 (sleep mutex:vnode_free_list) 1 0 86 0 230 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:5263 (sleep mutex:vnode interlock) 0 0 19 0 229 0 0 0 0 /usr/src/sys/vm/vm_page.c:1437 (sleep mutex:vm page) 0 0 24 0 221 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 19) 0 0 18 0 209 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 20) 1 0 125 8 205 0 0 0 19 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 11) 0 0 23 0 197 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 13) 1 2 43 2 192 0 0 0 1 /usr/src/sys/vm/vm_fault.c:503 (sleep mutex:vm page) 7 0 196 0 186 1 0 0 0 /usr/src/sys/vm/vm_fault.c:373 (rw:vm object) 1 0 92 10 177 0 0 0 25 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 10) 14 0 210 6 176 1 0 0 15 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 2) 0 0 20 0 174 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 14) 0 0 17 0 174 0 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:4460 (rw:pmap pv list) 1 1 112 7 171 0 0 0 18 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 9) 7 5 102 5 166 0 0 0 1 /usr/src/sys/kern/kern_mutex.c:190 (sleep mutex:ehci1) 3 1 41 1 162 0 0 0 1 /usr/src/sys/kern/subr_taskqueue.c:284 (sleep mutex:mlx4_en) 2 2 180 2 162 1 0 0 1 /usr/src/sys/kern/subr_taskqueue.c:467 (sleep mutex:mlx4_en) 6 0 205 0 160 1 0 0 0 /usr/src/sys/dev/usb/usb_transfer.c:2411 (sleep mutex:ehci1) 2 0 81 0 160 0 0 0 0 /usr/src/sys/dev/usb/usb_transfer.c:2266 (sleep mutex:ehci1) 0 0 14 0 156 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:509 (sx:zio->io_lock) 33 0 381 0 150 2 0 0 0 /usr/src/sys/vm/vm_pageout.c:336 (sleep mutex:vm page) 1 0 107 8 149 0 0 0 18 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 5) 2 2 107 40 145 0 0 0 49 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 2) 1 0 86 3 142 0 0 0 9 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 3) 1 2 73 56 141 0 0 0 57 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 11) 1 0 87 7 136 0 0 0 18 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 7) 1 0 80 7 135 0 0 0 17 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 8) 30 0 333 0 133 2 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:5879 (rw:pmap pv list) 1 2 107 31 133 0 0 0 38 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 4) 4 0 141 3 131 1 0 0 9 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 6) 0 0 18 0 125 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 17) 22 0 768 0 124 6 0 0 0 /usr/src/sys/dev/usb/controller/ehci.c:1465 (sleep mutex:ehci1) 8 0 327 0 124 2 0 0 0 /usr/src/sys/dev/usb/usb_transfer.c:1837 (sleep mutex:ehci1) 4 0 184 0 124 1 0 0 0 /usr/src/sys/dev/usb/controller/ehci.c:1465 (sleep mutex:ehci0) 3 0 37 0 124 0 0 0 0 /usr/src/sys/x86/x86/io_apic.c:259 (spin mutex:icu) 1 0 37 0 124 0 0 0 0 /usr/src/sys/dev/usb/usb_transfer.c:2995 (sleep mutex:ehci1) 0 0 30 0 124 0 0 0 0 /usr/src/sys/dev/usb/usb_transfer.c:2402 (sleep mutex:ehci1) 0 0 23 0 124 0 0 0 0 /usr/src/sys/x86/x86/io_apic.c:276 (spin mutex:icu) 1 0 97 0 123 0 0 0 0 /usr/src/sys/kern/kern_mutex.c:190 (sleep mutex:mlx4_en) 2 0 24 0 121 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 22) 1 1 81 17 121 0 0 0 42 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 1) 0 0 24 0 121 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 0) 1 0 82 1 118 0 0 0 2 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 23) 0 0 23 0 118 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/rrwlock.c:242 (sx:rrl->rr_lock) 2 0 100 0 117 0 0 0 0 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 23) 1 0 69 5 116 0 0 0 14 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 6) 0 0 5 0 116 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/rrwlock.c:165 (sx:rrl->rr_lock) 2 2 93 27 115 0 0 0 36 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 8) 9 0 139 4 114 1 0 0 12 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 4) 2 0 60 0 114 0 0 0 0 /usr/src/sys/vm/vm_fault.c:409 (rw:vm object) 1 0 21 0 114 0 0 0 0 /usr/src/sys/vm/vm_object.c:2244 (sleep mutex:vm page) 2 4 91 39 113 0 0 0 37 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 6) 32 7 466 31 108 4 0 0 12 /usr/src/sys/dev/usb/usb_transfer.c:2265 (sleep mutex:USB device mutex) 2 0 109 0 108 1 0 0 0 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 19) 1 0 77 1 108 0 0 0 3 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 19) 1 0 35 0 108 0 0 0 0 /usr/src/sys/vm/uma_core.c:2206 (sleep mutex:mlx4_cmd) 0 0 15 0 108 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 0) 4 0 109 0 107 1 0 0 0 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 22) 0 0 35 0 107 0 0 0 0 /usr/src/sys/vm/vm_object.c:763 (sleep mutex:vm page) 0 0 34 0 107 0 0 0 0 /usr/src/sys/vm/uma_core.c:2724 (sleep mutex:mlx4_cmd) 1 1 83 3 105 0 0 0 3 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 15) 1 0 82 0 104 0 0 0 1 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 22) 2 0 89 0 103 0 0 0 0 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 15) 0 3 55 29 103 0 0 0 38 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 7) 0 2 48 28 103 0 0 0 28 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 9) 6 0 122 0 102 1 0 0 1 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 21) 2 0 93 0 102 0 0 0 0 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 21) 1 2 76 6 101 0 0 0 4 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 21) 1 0 83 0 100 0 0 0 0 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 16) 1 0 76 1 100 0 0 0 2 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 16) 1 0 84 0 98 0 0 0 0 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 20) 1 0 55 3 98 0 0 0 8 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 4) 1 1 85 1 97 0 0 0 2 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 12) 3 0 91 0 96 0 0 0 0 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 12) 1 1 67 1 96 0 0 0 1 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 20) 3 0 83 0 94 0 0 0 0 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 13) 2 0 83 0 93 0 0 0 1 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 14) 1 2 74 4 92 0 0 0 2 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 13) 2 0 89 0 91 0 0 0 0 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 14) 8 0 109 0 90 1 0 0 1 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 13) 0 0 17 0 90 0 0 0 0 /usr/src/sys/kern/kern_sig.c:1027 (sleep mutex:process lock) 23 0 877 0 88 9 0 0 0 /usr/src/sys/dev/usb/usb_transfer.c:2265 (sleep mutex:ums lock) 8 0 100 5 87 1 0 0 8 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 5) 0 2 40 35 87 0 0 0 36 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 10) 3 0 119 0 85 1 0 0 1 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 17) 4 0 112 1 84 1 0 0 2 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 19) 1 1 49 20 84 0 0 0 25 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 3) 3 2 68 28 83 0 0 0 29 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 5) 5 0 104 1 82 1 0 0 3 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 8) 4 1 100 1 82 1 0 0 1 /usr/src/sys/kern/kern_mutex.c:190 (sleep mutex:process lock) 9 0 35 0 81 0 0 0 0 /usr/src/sys/vm/vm_map.c:2931 (rw:vm object) 3 0 100 1 81 1 0 0 3 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 16) 1211 0 52319 0 80 653 0 0 0 /usr/src/sys/modules/mlxen/../../ofed/drivers/net/mlx4/en_netdev.c:1150 (sx:lnxmtx) 208 0 14470 0 80 180 0 0 0 /usr/src/sys/modules/mlx4/../../ofed/drivers/net/mlx4/main.c:2655 (sx:lnxmtx) 11 0 373 0 80 4 0 0 0 /usr/src/sys/modules/mlxen/../../ofed/drivers/net/mlx4/en_port.c:182 (sleep mutex:lnxspin) 5 1 107 1 80 1 0 0 1 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 23) 3 0 82 0 80 1 0 0 0 /usr/src/sys/modules/mlxen/../../ofed/drivers/net/mlx4/en_netdev.c:1175 (sx:lnxmtx) 2 0 95 0 80 1 0 0 0 /usr/src/sys/modules/mlxen/../../ofed/drivers/net/mlx4/en_port.c:530 (sleep mutex:lnxspin) 1 3 35 31 80 0 0 0 33 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 8) 1 0 53 0 80 0 0 0 0 /usr/src/sys/kern/kern_resource.c:616 (spin mutex:pstatl) 2 3 77 26 79 0 0 0 26 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 10) 1 0 8 0 79 0 0 0 1 /usr/src/sys/vm/vm_page.c:3185 (sleep mutex:vm inactive pagequeue) 4 5 147 8 78 1 0 0 2 /usr/src/sys/vm/vm_fault.c:793 (rw:vm object) 1 0 20 0 78 0 0 0 0 /usr/src/sys/vm/vm_fault.c:139 (sleep mutex:vm page) 0 0 4 0 78 0 0 0 0 /usr/src/sys/vm/vm_fault.c:914 (rw:vm object) 3 0 81 0 77 1 0 0 0 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 22) 2 2 59 18 77 0 0 0 22 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 7) 0 0 6 0 77 0 0 0 0 /usr/src/sys/vm/vm_page.c:2796 (sleep mutex:vm inactive pagequeue) 6 0 83 0 76 1 0 0 0 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 12) 1 2 44 15 76 0 0 0 21 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 5) 1 0 44 1 76 0 0 0 3 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 2) 5 0 91 3 75 1 0 0 8 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 3) 4 0 70 0 75 0 0 0 1 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 20) 2 0 72 0 75 0 0 0 0 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 17) 1 1 71 1 75 0 0 0 1 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 17) 3 0 76 0 72 1 0 0 1 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 18) 0 0 7 0 72 0 0 0 1 /usr/src/sys/dev/usb/usb_transfer.c:2013 (sleep mutex:ehci1) 2 0 80 1 70 1 0 0 2 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 14) 1 0 18 0 70 0 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:4482 (rw:pmap pv list) 2 0 74 0 67 1 0 0 0 /usr/src/sys/net/bpf.c:2206 (rw:bpf interface lock) 1507 0 8304 0 66 125 0 0 0 /usr/src/sys/dev/usb/usb_device.c:2798 (sx:USB suspend and resume SX lock) 1 0 18 0 66 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_acl.c:2100 (sx:zp->z_acl_lock) 3 0 80 0 65 1 0 0 1 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 15) 0 0 12 0 65 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 22) 1 1 34 21 64 0 0 0 26 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 6) 24 0 217 0 60 3 0 0 0 /usr/src/sys/kern/vfs_subr.c:2523 (lockmgr:zfs) 3 0 67 0 57 1 0 0 1 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 10) 1 4 40 27 57 0 0 0 23 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 3) 1 0 7 0 57 0 0 0 0 /usr/src/sys/vm/vm_glue.c:756 (sleep mutex:process lock) 0 0 6 0 57 0 0 0 0 /usr/src/sys/kern/sys_generic.c:1777 (sleep mutex:select mtxpool) 2 0 22 0 56 0 0 0 0 /usr/src/sys/kern/vfs_cache.c:585 (rw:Name Cache) 12 0 200 0 54 3 0 0 0 /usr/src/sys/dev/usb/usb_request.c:544 (sleep mutex:USB device mutex) 9 8 229 59 54 4 1 0 12 /usr/src/sys/kern/kern_condvar.c:150 (sleep mutex:Giant) 2 5 32 57 54 0 1 0 37 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 18) 1 0 50 0 54 0 0 0 0 /usr/src/sys/kern/kern_mutex.c:190 (sleep mutex:USB device mutex) 0 0 6 0 54 0 0 0 0 /usr/src/sys/dev/usb/usb_transfer.c:3102 (sleep mutex:USB device mutex) 0 1 3 1 53 0 0 0 3 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 5) 0 0 8 0 53 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 0) 0 0 8 0 53 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 0) 0 0 4 2 53 0 0 0 4 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 5) 1 0 3 0 51 0 0 0 0 /usr/src/sys/vm/swap_pager.c:307 (sleep mutex:ui_vmsize) 0 0 3 0 51 0 0 0 0 /usr/src/sys/vm/swap_pager.c:301 (sleep mutex:swapdev) 2 0 53 0 50 1 0 0 0 /usr/src/sys/kern/kern_mutex.c:190 (sleep mutex:msgbuf lock) 0 0 4 0 49 0 0 0 2 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 9) 0 0 4 0 49 0 0 0 0 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 9) 3 0 147 0 48 3 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/txg.c:367 (sx:tx->tx_cpu[c].tc_open_lock) 3 0 54 1 48 1 0 0 3 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 7) 2 3 20 54 48 0 1 0 39 /usr/src/sys/vm/uma_core.c:2208 (sleep mutex:mbuf) 0 0 10 0 48 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 17) 0 0 8 0 48 0 0 0 0 /usr/src/sys/kern/sys_generic.c:1708 (sleep mutex:select mtxpool) 0 0 3 0 48 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/txg.c:388 (sx:tx->tx_cpu[c].tc_lock) 3 0 60 0 47 1 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 18) 0 0 10 0 47 0 0 0 0 /usr/src/sys/kern/kern_sig.c:665 (sleep mutex:process lock) 0 0 4 0 47 0 0 0 0 /usr/src/sys/kern/kern_sig.c:667 (sleep mutex:sigacts) 20 0 187 0 46 4 0 0 0 /usr/src/sys/dev/random/fortuna.c:356 (sleep mutex:reseed mutex) 4 0 65 0 46 1 0 0 0 /usr/src/sys/dev/random/fortuna.c:412 (sleep mutex:reseed mutex) 0 0 8 0 46 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:2926 (sleep mutex:vnode interlock) 0 0 6 0 46 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:2627 (sleep mutex:vnode interlock) 0 0 4 0 46 0 0 0 0 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 11) 0 0 4 0 46 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c:4288 (sx:zfsvfs->z_teardown_inactive_lock) 0 0 2 0 46 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:2541 (sleep mutex:vnode interlock) 3 4 43 34 45 0 0 0 27 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 23) 3 0 62 0 45 1 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 20) 2 2 45 21 45 1 0 0 18 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 9) 1 0 23 0 44 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/multilist.c:257 (sx:mls->mls_lock) 9 0 99 0 43 2 0 0 0 /usr/src/sys/kern/subr_turnstile.c:541 (spin mutex:turnstile chain) 3 0 64 0 43 1 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 17) 3 4 41 25 42 0 0 0 18 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 15) 2 3 50 21 42 1 0 0 14 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 11) 2 0 11 0 42 0 0 0 0 /usr/src/sys/vm/vm_fault.c:1186 (rw:vm object) 1 1 18 2 42 0 0 0 2 /usr/src/sys/kern/kern_mutex.c:190 (sleep mutex:ehci0) 0 0 6 0 42 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 8) 0 0 3 0 42 0 0 0 0 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 10) 0 0 3 0 42 0 0 0 0 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 10) 0 0 2 0 42 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:2668 (sleep mutex:vnode interlock) 0 0 10 0 41 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 20) 2 0 48 0 40 1 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 22) 0 1 3 2 39 0 0 0 3 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 8) 0 0 3 0 39 0 0 0 1 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 8) 15 0 139 0 37 3 0 0 0 /usr/src/sys/kern/tty_ttydisc.c:480 (sleep mutex:ttymtx) 2 3 29 23 37 0 0 0 15 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 21) 1 0 19 0 37 0 0 0 0 /usr/src/sys/netinet/ip_input.c:690 (rw:if_addr_lock) 0 2 4 9 37 0 0 0 9 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 22) 0 1 4 5 37 0 0 0 6 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 22) 0 0 7 0 37 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 1) 0 0 3 0 37 0 0 0 3 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 4) 0 0 3 0 37 0 0 0 0 /usr/src/sys/vm/vm_object.c:435 (rw:vm object) 8 0 18 0 36 0 0 0 0 /usr/src/sys/dev/usb/usb_transfer.c:2266 (sleep mutex:ehci0) 5 0 62 0 36 1 0 0 0 /usr/src/sys/dev/usb/usb_transfer.c:1891 (sleep mutex:ehci1) 3 0 41 0 36 1 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 23) 2 0 36 0 36 1 0 0 0 /usr/src/sys/dev/usb/usb_transfer.c:2411 (sleep mutex:ehci0) 1 0 10 0 36 0 0 0 0 /usr/src/sys/vm/vm_map.c:3171 (rw:vm object) 1 0 10 0 36 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 14) 0 0 5 0 36 0 0 0 0 /usr/src/sys/dev/usb/usb_hub.c:2371 (sleep mutex:ehci0) 0 0 4 0 36 0 0 0 0 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 19) 0 0 4 0 36 0 0 0 0 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 19) 0 0 4 0 36 0 0 0 0 /usr/src/sys/dev/usb/usb_transfer.c:2013 (sleep mutex:ehci0) 0 0 3 1 36 0 0 0 3 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 6) 0 0 3 0 36 0 0 0 0 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 4) 3 0 49 0 35 1 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 21) 3 0 11 0 35 0 0 0 0 /usr/src/sys/kern/kern_sysctl.c:164 (sleep mutex:Giant) 1 0 17 0 35 0 0 0 0 /usr/src/sys/vm/vm_object.c:1912 (sleep mutex:vm page) 0 2 9 37 35 0 1 0 27 /usr/src/sys/vm/uma_core.c:2726 (sleep mutex:mbuf) 0 1 3 2 35 0 0 0 2 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 20) 0 0 5 1 35 0 0 0 2 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 21) 0 0 3 0 35 0 0 0 1 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 6) 0 0 2 0 35 0 0 0 1 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 3) 7 0 96 0 34 2 0 0 0 /usr/src/sys/kern/sys_pipe.c:1209 (sleep mutex:pipe mutex) 3 6 30 41 34 0 1 0 20 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 13) 3 0 16 0 34 0 0 0 0 /usr/src/sys/kern/tty_outq.c:244 (sleep mutex:ttymtx) 2 4 26 8 34 0 0 0 5 /usr/src/sys/kern/sys_pipe.c:1053 (sleep mutex:pipe mutex) 2 0 36 0 34 1 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 15) 1 14 19 88 34 0 2 0 19 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 14) 0 1 4 1 34 0 0 0 1 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 21) 0 0 3 0 34 0 0 0 1 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 2) 0 0 3 0 34 0 0 0 0 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 11) 1525 0 15102 0 33 457 0 0 0 /usr/src/sys/dev/usb/usb_device.c:2751 (sx:USB config SX lock) 15 0 42 0 33 1 0 0 0 /usr/src/sys/dev/usb/usb_device.c:2752 (sx:USB suspend and resume SX lock) 2 0 46 0 33 1 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 16) 2 0 44 0 33 1 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 12) 0 0 9 0 33 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 23) 0 0 2 0 33 0 0 0 1 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 2) 0 0 2 0 33 0 0 0 0 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 20) 4 1 11 3 32 0 0 0 3 /usr/src/sys/vm/vm_fault.c:1193 (rw:vm object) 3 2 24 3 32 0 0 0 2 /usr/src/sys/kern/sys_pipe.c:708 (sleep mutex:pipe mutex) 1 0 18 0 32 0 0 0 0 /usr/src/sys/kern/sys_pipe.c:671 (sleep mutex:pipe mutex) 0 0 13 0 32 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 13) 0 0 11 0 32 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 20) 0 0 3 0 32 0 0 0 0 /usr/src/sys/kern/kern_umtx.c:3747 (sleep mutex:umtxshm) 0 0 2 0 32 0 0 0 1 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 7) 0 0 2 0 32 0 0 0 0 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 3) 2 0 44 0 31 1 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 14) 0 0 13 0 31 0 0 0 0 /usr/src/sys/vm/vm_object.c:1605 (sleep mutex:vm page) 10 0 44 1 30 1 0 0 4 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 9) 4 1 43 11 30 1 0 0 11 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 19) 4 0 40 0 30 1 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 13) 3 3 46 20 30 1 0 0 15 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 18) 2 2 30 8 30 1 0 0 11 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 17) 0 1 9 1 30 0 0 0 1 /usr/src/sys/kern/tty_pts.c:392 (sleep mutex:ttymtx) 0 0 11 0 30 0 0 0 0 /usr/src/sys/kern/kern_kthread.c:214 (sleep mutex:process lock) 2 0 36 0 29 1 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 19) 1 3 18 26 29 0 0 0 18 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 22) 4 0 19 0 28 0 0 0 0 /usr/src/sys/kern/kern_mutex.c:190 (sleep mutex:pipe mutex) 1 0 8 0 28 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sa.c:1891 (sx:hdl->sa_lock) 0 0 3 0 28 0 0 0 0 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 18) 0 0 3 0 28 0 0 0 0 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 18) 0 0 2 0 28 0 0 0 1 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 7) 3 0 16 0 27 0 0 0 0 /usr/src/sys/kern/tty.c:171 (sleep mutex:ttymtx) 2 0 7 0 27 0 0 0 0 /usr/src/sys/kern/kern_conf.c:72 (sleep mutex:cdev) 1 0 16 0 27 0 0 0 0 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 1) 12 0 59 0 26 2 0 0 0 /usr/src/sys/kern/vfs_lookup.c:530 (lockmgr:zfs) 1 2 17 12 26 0 0 0 10 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 20) 1 0 3 0 26 0 0 0 0 /usr/src/sys/kern/vfs_lookup.c:235 (sx:filedesc structure) 0 3 6 20 26 0 0 0 22 /usr/src/sys/vm/uma_core.c:2726 (sleep mutex:mbuf_jumbo_page) 0 0 7 0 26 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 12) 0 0 4 0 26 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:3486 (sx:zio->io_lock) 0 0 4 0 26 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:3198 (sx:zio->io_lock) 0 0 2 0 26 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/ddt.c:637 (sx:ddt->ddt_lock) 0 0 8 0 25 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 22) 0 0 4 0 25 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 7) 8 1 79 8 24 3 0 0 12 /usr/src/sys/kern/subr_turnstile.c:598 (spin mutex:turnstile lock) 1 0 8 0 24 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 16) 1 0 7 0 24 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:896 (spin mutex:td_contested) 0 5 2 5 24 0 0 0 1 /usr/src/sys/vm/vm_object.c:1266 (rw:vm object) 0 4 1 4 24 0 0 0 1 /usr/src/sys/vm/vm_object.c:1233 (rw:vm object) 0 3 8 24 24 0 1 0 20 /usr/src/sys/vm/uma_core.c:2208 (sleep mutex:mbuf_jumbo_page) 0 2 3 5 24 0 0 0 3 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 16) 0 0 13 0 24 0 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 11) 0 0 7 0 24 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:836 (spin mutex:td_contested) 0 0 5 1 24 0 0 0 2 /usr/src/sys/kern/kern_proc.c:1030 (spin mutex:sched lock 0) 0 0 5 0 24 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:712 (spin mutex:td_contested) 0 0 2 0 24 0 0 0 0 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 14) 0 0 2 0 24 0 0 0 0 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 14) 0 0 1 0 24 0 0 0 0 /usr/src/sys/vm/vm_map.c:4084 (rw:vm object) 5 0 72 0 23 3 0 0 0 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 18) 4 0 56 0 23 2 0 0 0 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 14) 1 0 15 0 23 0 0 0 0 /usr/src/sys/kern/uipc_socket.c:3026 (sleep mutex:so_snd) 0 2 11 8 23 0 0 0 8 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 4) 0 2 3 7 23 0 0 0 4 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 13) 0 1 3 2 23 0 0 0 2 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 15) 0 0 11 0 23 0 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 8) 0 0 8 0 23 0 0 0 0 /usr/src/sys/kern/uipc_socket.c:3027 (sleep mutex:so_rcv) 0 0 5 0 23 0 0 0 1 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 9) 0 0 5 0 23 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 15) 0 0 2 0 23 0 0 0 0 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 15) 0 0 1 0 23 0 0 0 0 /usr/src/sys/vm/swap_pager.c:227 (sleep mutex:ui_vmsize) 0 0 1 0 23 0 0 0 0 /usr/src/sys/vm/swap_pager.c:211 (sleep mutex:swapdev) 1 0 12 0 22 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:3498 (sx:zio->io_lock) 1 0 10 0 22 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 14) 0 1 3 2 22 0 0 0 3 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 13) 0 1 2 2 22 0 0 0 2 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 12) 0 1 2 1 22 0 0 0 2 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 16) 0 0 11 0 22 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 16) 0 0 9 0 22 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 15) 0 0 4 0 22 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:1582 (sx:zio->io_lock) 0 1 2 1 21 0 0 0 1 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 12) 0 0 10 0 21 0 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 4) 0 0 2 0 21 0 0 0 0 /usr/src/sys/vm/vm_mmap.c:1482 (sleep mutex:process lock) 0 0 2 0 21 0 0 0 0 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 23) 0 0 2 0 21 0 0 0 0 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 23) 71 0 423 0 20 21 0 0 0 /usr/src/sys/vm/vm_pageout.c:1199 (sleep mutex:vm active pagequeue) 26 0 82 0 20 4 0 0 0 /usr/src/sys/vm/vm_object.c:1693 (rw:vm object) 4 0 35 0 20 1 0 0 0 /usr/src/sys/kern/kern_mutex.c:190 (sleep mutex:Giant) 4 0 22 0 20 1 0 0 0 /usr/src/sys/netinet6/mld6.c:1746 (sleep mutex:mld_mtx) 3 0 14 0 20 0 0 0 0 /usr/src/sys/netinet/igmp.c:2186 (sleep mutex:igmp_mtx) 1 0 14 0 20 0 0 0 0 /usr/src/sys/kern/sys_pipe.c:1435 (sleep mutex:pipe mutex) 1 0 10 0 20 0 0 0 0 /usr/src/sys/vm/vm_pageout.c:1552 (sleep mutex:vm page free queue) 1 0 6 0 20 0 0 0 0 /usr/src/sys/netinet/tcp_timewait.c:691 (rw:tcptw) 0 0 9 0 20 0 0 0 0 /usr/src/sys/vm/vm_pageout.c:951 (sleep mutex:vm inactive pagequeue) 0 0 7 0 20 0 0 0 0 /usr/src/sys/netinet6/frag6.c:741 (sleep mutex:ip6qlock) 40 0 112 0 19 5 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:5022 (sleep mutex:pmap) 40 0 112 0 19 5 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:5021 (sleep mutex:pmap) 30 0 109 0 19 5 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:5018 (sleep mutex:pmap) 30 0 106 0 19 5 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:5019 (sleep mutex:pmap) 11 0 33 0 19 1 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:3776 (sleep mutex:pmap) 3 0 8 0 19 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 23) 1 0 11 0 19 0 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 7) 1 0 5 0 19 0 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:484 (sleep mutex:invlgn) 0 1 9 7 19 0 0 0 9 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 12) 0 0 10 0 19 0 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 5) 0 0 1 0 19 0 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:452 (sleep mutex:invlgn) 8 2 35 8 18 1 0 0 6 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 12) 7 0 49 0 18 2 0 0 0 /usr/src/sys/dev/usb/usb_transfer.c:1891 (sleep mutex:ehci0) 7 0 43 0 18 2 0 0 0 /usr/src/sys/dev/usb/usb_transfer.c:1837 (sleep mutex:ehci0) 2 0 25 0 18 1 0 0 0 /usr/src/sys/kern/sys_generic.c:1869 (sleep mutex:sellck) 1 0 7 0 18 0 0 0 0 /usr/src/sys/kern/kern_mutex.c:190 (sleep mutex:sellck) 0 0 6 0 18 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 17) 0 0 5 0 18 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 21) 0 0 2 0 18 0 0 0 0 /usr/src/sys/dev/usb/usb_transfer.c:2995 (sleep mutex:ehci0) 0 0 1 0 18 0 0 0 0 /usr/src/sys/dev/usb/usb_transfer.c:2402 (sleep mutex:ehci0) 0 0 7 0 17 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 18) 4 0 35 0 16 2 0 0 0 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 22) 4 0 14 0 16 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:2401 (sx:db->db_mtx) 3 2 29 8 16 1 0 0 8 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 16) 1 0 7 0 16 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c:523 (sx:scl->scl_lock) 1 0 7 0 16 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c:496 (sx:scl->scl_lock) 0 0 1 0 16 0 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:3121 (sleep mutex:pmap pv chunk list) 0 0 1 0 16 0 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:3046 (sleep mutex:pmap pv chunk list) 4 0 34 0 15 2 0 0 0 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 16) 2 0 7 0 15 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 18) 1 0 16 0 15 1 0 0 0 /usr/src/sys/vm/vm_map.c:1505 (sx:vm map (user)) 1 0 8 0 15 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 19) 0 0 9 0 15 0 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 3) 0 0 7 0 15 0 0 0 0 /usr/src/sys/kern/subr_vmem.c:1331 (sleep mutex:kmem arena) 0 0 3 0 15 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 11) 0 0 1 0 15 0 0 0 0 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 17) 0 0 1 0 15 0 0 0 0 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 17) 8 0 43 0 14 3 0 0 0 /usr/src/sys/kern/sys_generic.c:1824 (sleep mutex:select mtxpool) 7 0 25 0 14 1 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:4067 (sleep mutex:pmap) 0 0 7 0 14 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 21) 0 0 7 0 14 0 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 6) 0 0 2 0 14 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 10) 0 0 1 0 14 0 0 0 0 /usr/src/sys/vm/vm_object.c:659 (rw:vm object) 0 0 1 0 14 0 0 0 0 /usr/src/sys/vm/vm_map.c:1262 (rw:vm object) 3 0 35 0 13 2 0 0 0 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 12) 0 2 2 4 13 0 0 0 4 /usr/src/sys/kern/subr_turnstile.c:730 (spin mutex:sched lock 0) 0 0 8 0 13 0 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 10) 0 0 4 0 13 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 19) 0 0 3 0 13 0 0 0 0 /usr/src/sys/vm/uma_core.c:2206 (sleep mutex:tcpreass) 499 5 4728 5 12 394 0 0 1 /usr/src/sys/kern/kern_proc.c:1439 (sx:proctree) 498 0 4722 0 12 393 0 0 0 /usr/src/sys/kern/kern_proc.c:1440 (sx:allproc) 30 0 76 0 12 6 0 0 0 /usr/src/sys/vm/vm_map.c:1875 (rw:vm object) 22 0 58 0 12 4 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:4636 (sleep mutex:pmap) 15 4 66 6 12 5 0 0 2 /usr/src/sys/dev/usb/usb_device.c:2804 (sleep mutex:Giant) 2 4 21 12 12 1 1 0 7 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 14) 1 14 7 18 12 0 1 0 2 /usr/src/sys/dev/usb/usb_device.c:2758 (sleep mutex:Giant) 1 0 5 0 12 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 12) 1 0 3 0 12 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:720 (sx:db->db_mtx) 0 0 4 0 12 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 1) 0 0 4 0 12 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:136 (sx:h->hash_mutexes[i]) 0 0 3 0 12 0 0 0 0 /usr/src/sys/kern/kern_proc.c:1030 (spin mutex:sched lock 11) 0 0 2 0 12 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 6) 0 0 1 0 12 0 0 0 0 /usr/src/sys/vm/vm_map.c:4112 (rw:vm object) 0 0 1 0 12 0 0 0 0 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 1) 0 0 1 0 12 0 0 0 0 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 1) 0 0 1 0 12 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/txg.c:808 (sx:tl->tl_lock) 0 0 1 0 12 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c:5059 (sx:buf->b_evict_lock) 0 0 0 0 12 0 0 0 0 /usr/src/sys/vm/vm_fault.c:208 (sleep mutex:vm page) 13 0 23 0 11 2 0 0 0 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 11) 6 0 35 0 11 3 0 0 0 /usr/src/sys/kern/sys_generic.c:1833 (sleep mutex:sellck) 6 0 35 0 11 3 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c:3752 (sx:arc_reclaim_lock) 5 0 26 0 11 2 0 0 0 /usr/src/sys/vm/vm_mmap.c:550 (sx:vm map (user)) 3 0 23 0 11 2 0 0 0 /usr/src/sys/kern/kern_sx.c:195 (sx:arc_reclaim_lock) 1 2 6 8 11 0 0 0 5 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 16) 0 0 6 0 11 0 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 9) 26 0 87 0 10 8 0 0 0 /usr/src/sys/vm/vm_object.c:587 (rw:vm object) 7 0 24 0 10 2 0 0 0 /usr/src/sys/kern/kern_sx.c:195 (sx:l2arc_feed_thr_lock) 5 2 23 8 10 2 0 0 6 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 22) 5 0 33 0 10 3 0 0 0 /usr/src/sys/kern/kern_mutex.c:190 (sleep mutex:Syncer mtx) 5 0 31 0 10 3 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/trim_map.c:564 (sx:spa->spa_trim_lock) 4 0 21 0 10 2 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c:3819 (sx:arc_user_evicts_lock) 4 0 19 0 10 1 0 0 0 /usr/src/sys/kern/subr_eventhandler.c:251 (sleep mutex:eventhandler) 3 0 15 0 10 1 0 0 0 /usr/src/sys/kern/vfs_bio.c:3038 (sleep mutex:buffer daemon lock) 2 0 18 0 10 1 0 0 0 /usr/src/sys/kern/kern_mutex.c:190 (sleep mutex:MPT2SAS lock) 1 0 13 0 10 1 0 0 0 /usr/src/sys/kern/vfs_subr.c:1066 (sleep mutex:vnode_free_list) 1 0 7 0 10 0 0 0 0 /usr/src/sys/kern/vfs_bio.c:713 (rw:needsbuffer lock) 1 0 6 0 10 0 0 0 0 /usr/src/sys/kern/tty_pts.c:121 (sleep mutex:ttymtx) 1 0 6 0 10 0 0 0 0 /usr/src/sys/kern/kern_sx.c:195 (sx:arc_user_evicts_lock) 1 0 6 0 10 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/trim_map.c:455 (sx:tm->tm_lock) 1 0 5 0 10 0 0 0 0 /usr/src/sys/netipsec/key.c:4424 (sleep mutex:acqtree) 1 0 5 0 10 0 0 0 0 /usr/src/sys/netipsec/key.c:4298 (sleep mutex:sahtree) 1 0 5 0 10 0 0 0 0 /usr/src/sys/kern/kern_mutex.c:190 (sleep mutex:buffer daemon lock) 1 0 5 0 10 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/trim_map.c:500 (sx:tm->tm_lock) 1 0 4 0 10 0 0 0 0 /usr/src/sys/fs/nfsserver/nfs_nfsdstate.c:1075 (sleep mutex:nfs_state_mutex) 1 0 3 0 10 0 0 0 0 /usr/src/sys/kern/vfs_bio.c:441 (sleep mutex:dirty buf lock) 0 0 5 0 10 0 0 0 0 /usr/src/sys/kern/kern_sx.c:195 (sx:spa->spa_trim_lock) 0 0 4 0 10 0 0 0 0 /usr/src/sys/netinet6/nd6.c:913 (rw:nd6) 0 0 2 0 10 0 0 0 0 /usr/src/sys/vm/uma_core.c:2724 (sleep mutex:tcpreass) 0 0 2 0 10 0 0 0 0 /usr/src/sys/netipsec/key.c:4442 (sleep mutex:spacqtree) 0 0 2 0 10 0 0 0 0 /usr/src/sys/kern/vfs_bio.c:763 (sleep mutex:runningbufspace lock) 0 0 2 0 10 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 2) 0 0 2 0 10 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c:7012 (sx:l2arc_dev_mtx) 0 0 2 0 10 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c:3369 (sx:arc_user_evicts_lock) 0 0 1 0 10 0 0 0 0 /usr/src/sys/vm/vnode_pager.c:121 (rw:vm object) 0 0 1 0 10 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 3) 0 0 1 0 10 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 2) 1 0 8 0 9 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:210 (spin mutex:sched lock 0) 0 0 2 0 9 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 5) 33 0 75 0 8 9 0 0 0 /usr/src/sys/vm/vm_map.c:1458 (sx:vm map (user)) 7 0 19 0 8 2 0 0 2 /usr/src/sys/kern/kern_sx.c:195 (sx:tx->tx_sync_lock) 6 0 14 0 8 1 0 0 0 /usr/src/sys/kern/kern_mutex.c:190 (sleep mutex:CAM device lock) 6 0 14 0 8 1 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:139 (sx:db->db_mtx) 3 0 15 0 8 1 0 0 0 /usr/src/sys/kern/imgact_elf.c:480 (sx:vm map (user)) 1 0 3 0 8 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:714 (sx:dn->dn_struct_rwlock) 0 1 5 2 8 0 0 0 3 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 2) 0 0 4 0 8 0 0 0 0 /usr/src/sys/vm/uma_core.c:2206 (sleep mutex:32 Bucket) 0 0 4 0 8 0 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 2) 0 0 2 0 8 0 0 0 0 /usr/src/sys/kern/sys_generic.c:777 (sx:filedesc structure) 0 0 1 0 8 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:2006 (sleep mutex:Syncer mtx) 0 0 1 0 8 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 4) 0 0 1 0 8 0 0 0 0 /usr/src/sys/kern/kern_exit.c:332 (sleep mutex:process_exit) 0 0 0 0 8 0 0 0 0 /usr/src/sys/vm/vnode_pager.c:225 (rw:vm object) 0 0 0 0 8 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:527 (sx:zio->io_lock) 0 0 0 0 8 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dnode.c:1230 (sx:dn->dn_mtx) 3 0 17 0 7 2 0 0 0 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 20) 0 0 2 0 7 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 13) 0 0 1 0 7 0 0 0 0 /usr/src/sys/vm/vm_object.c:2080 (rw:vm object) 0 0 0 0 7 0 0 0 0 /usr/src/sys/vm/vm_reserv.c:1096 (sleep mutex:vm page free queue) 355 0 2036 0 6 339 0 0 0 /usr/src/sys/kern/kern_sysctl.c:1971 (sx:sysctl mem) 9 0 26 0 6 4 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:3708 (rw:pmap pv list) 7 0 44 0 6 7 0 0 0 /usr/src/sys/vm/vm_map.c:2335 (sx:vm map (user)) 7 0 24 0 6 4 0 0 0 /usr/src/sys/x86/x86/mp_x86.c:1402 (spin mutex:smp rendezvous) 5 2 14 4 6 2 0 0 2 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 20) 5 0 31 0 6 5 0 0 0 /usr/src/sys/vm/vm_object.c:2213 (rw:vm object) 3 0 14 0 6 2 0 0 0 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 13) 3 0 10 0 6 1 0 0 0 /usr/src/sys/dev/usb/usb_request.c:493 (sleep mutex:ehci0) 2 0 12 0 6 2 0 0 0 /usr/src/sys/vm/vm_map.c:2540 (sx:vm map (user)) 2 0 6 0 6 1 0 0 0 /usr/src/sys/kern/subr_taskqueue.c:467 (spin mutex:mlx4_en_que) 1 0 8 0 6 1 0 0 0 /usr/src/sys/kern/vfs_vnops.c:1343 (lockmgr:zfs) 1 0 7 0 6 1 0 0 0 /usr/src/sys/dev/usb/usb_request.c:493 (sleep mutex:ehci1) 1 0 7 0 6 1 0 0 0 /usr/src/sys/dev/usb/controller/ehci.c:3900 (sleep mutex:ehci0) 1 0 6 0 6 1 0 0 1 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 1) 1 0 5 0 6 0 0 0 0 /usr/src/sys/kern/vfs_vnops.c:442 (lockmgr:zfs) 0 0 3 0 6 0 0 0 0 /usr/src/sys/kern/kern_synch.c:309 (spin mutex:mlx4_en_que) 0 0 2 0 6 0 0 0 0 /usr/src/sys/vm/vm_map.c:2642 (sx:vm map (user)) 0 0 2 0 6 0 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:4919 (sleep mutex:pmap) 0 0 1 0 6 0 0 0 0 /usr/src/sys/kern/subr_taskqueue.c:284 (spin mutex:mlx4_en_que) 0 0 1 0 6 0 0 0 0 /usr/src/sys/kern/kern_descrip.c:1813 (sx:filedesc structure) 0 0 1 0 6 0 0 0 0 /usr/src/sys/kern/kern_descrip.c:1229 (sx:filedesc structure) 0 0 1 0 6 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/bpobj.c:550 (sx:bpo->bpo_lock) 0 0 0 0 6 0 0 0 0 /usr/src/sys/kern/vfs_vnops.c:709 (sleep mutex:sleep mtxpool) 0 0 0 0 6 0 0 0 0 /usr/src/sys/kern/vfs_vnops.c:677 (sleep mutex:sleep mtxpool) 0 0 0 0 6 0 0 0 0 /usr/src/sys/kern/vfs_mount.c:443 (sleep mutex:struct mount mtx) 0 0 0 0 6 0 0 0 0 /usr/src/sys/kern/vfs_mount.c:433 (sleep mutex:struct mount mtx) 0 0 0 0 6 0 0 0 0 /usr/src/sys/kern/kern_proc.c:366 (sleep mutex:process group) 0 0 0 0 6 0 0 0 0 /usr/src/sys/kern/kern_exec.c:1079 (sleep mutex:process_exec) 12 0 20 0 5 4 0 0 0 /usr/src/sys/dev/mps/mpsvar.h:582 (sleep mutex:MPT2SAS lock) 3 0 10 0 5 2 0 0 0 /usr/src/sys/netinet/tcp_input.c:3053 (sleep mutex:so_rcv) 3 0 9 0 5 1 0 0 0 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 15) 0 2 2 8 5 0 1 0 5 /usr/src/sys/vm/uma_core.c:2726 (sleep mutex:mbuf_jumbo_9k) 0 0 0 0 5 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 11) 146 0 171 0 4 42 0 0 0 /usr/src/sys/kern/vfs_subr.c:2072 (lockmgr:syncer) 69 0 117 0 4 29 0 0 0 /usr/src/sys/kern/vfs_vnops.c:786 (lockmgr:zfs) 64 0 236 0 4 59 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:5435 (sleep mutex:pmap) 53 0 84 0 4 21 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu.c:411 (sx:dn->dn_struct_rwlock) 20 0 49 0 4 12 0 0 0 /usr/src/sys/vm/vm_map.c:3075 (sx:vm map (user)) 8 0 17 0 4 4 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu.c:146 (sx:dn->dn_struct_rwlock) 6 0 17 0 4 4 0 0 0 /usr/src/sys/kern/kern_exit.c:1146 (sx:proctree) 5 0 7 0 4 1 0 0 0 /usr/src/sys/vm/vm_map.c:1968 (sx:vm map (user)) 4 0 13 0 4 3 0 0 1 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 15) 4 0 11 0 4 2 0 0 0 /usr/src/sys/kern/vfs_subr.c:2088 (sleep mutex:Syncer mtx) 4 0 11 0 4 2 0 0 0 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 19) 4 0 8 0 4 2 0 0 0 /usr/src/sys/kern/kern_prot.c:405 (sx:proctree) 4 0 7 0 4 1 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dnode.c:1114 (sx:dn->dn_struct_rwlock) 3 2 7 3 4 1 0 0 2 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 13) 3 0 9 0 4 2 0 0 0 /usr/src/sys/net/route.c:457 (rw:rib head lock) 3 0 6 0 4 1 0 0 0 /usr/src/sys/kern/vfs_subr.c:2661 (lockmgr:zfs) 2 0 5 0 4 1 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c:2251 (sx:buf_hash_table.ht_locks[i].ht_lock) 2 0 5 0 4 1 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c:2245 (sx:buf->b_evict_lock) 2 0 4 0 4 1 0 0 1 /usr/src/sys/kern/kern_exit.c:920 (sleep mutex:process lock) 2 0 4 0 4 1 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:199 (sx:db->db_mtx) 2 0 3 0 4 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:592 (sx:db->db_mtx) 2 0 3 0 4 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:1913 (sx:dn->dn_dbufs_mtx) 1 1 1 1 4 0 0 0 2 /usr/src/sys/kern/tty.c:388 (sleep mutex:process lock) 1 0 5 0 4 1 0 0 0 /usr/src/sys/dev/usb/controller/ehci.c:3900 (sleep mutex:ehci1) 1 0 4 0 4 1 0 0 0 /usr/src/sys/netinet/if_ether.c:609 (rw:if_afdata) 1 0 4 0 4 1 0 0 0 /usr/src/sys/kern/kern_exec.c:981 (rw:vm object) 1 0 3 0 4 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:5318 (sleep mutex:vnode_free_list) 1 0 3 0 4 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:4145 (rw:bufobj interlock) 1 0 3 0 4 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:2056 (sleep mutex:vnode interlock) 1 0 3 0 4 0 0 0 0 /usr/src/sys/kern/tty.c:1728 (sleep mutex:ttymtx) 1 0 2 0 4 0 0 0 0 /usr/src/sys/kern/tty.c:1719 (sx:proctree) 1 0 2 0 4 0 0 0 0 /usr/src/sys/kern/kern_rangelock.c:219 (sleep mutex:vnode interlock) 1 0 2 0 4 0 0 0 0 /usr/src/sys/kern/kern_proc.c:630 (sleep mutex:process group) 1 0 2 0 4 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_rlock.c:444 (sx:zp->z_range_lock) 1 0 2 0 4 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c:2634 (sx:buf_hash_table.ht_locks[i].ht_lock) 1 0 1 0 4 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:464 (sx:zio->io_lock) 0 2 1 5 4 0 1 0 3 /usr/src/sys/vm/uma_core.c:2208 (sleep mutex:mbuf_jumbo_9k) 0 0 3 0 4 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:895 (spin mutex:sched lock 8) 0 0 2 0 4 0 0 0 0 /usr/src/sys/net/route.c:461 (sleep mutex:rtentry) 0 0 1 0 4 0 0 0 0 /usr/src/sys/vm/vm_glue.c:237 (rw:vm object) 0 0 1 0 4 0 0 0 0 /usr/src/sys/netinet/ip_output.c:712 (sleep mutex:rtentry) 0 0 1 0 4 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:533 (sleep mutex:struct mount mtx) 0 0 1 0 4 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:2662 (sleep mutex:vnode interlock) 0 0 1 0 4 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:2076 (rw:bufobj interlock) 0 0 1 0 4 0 0 0 0 /usr/src/sys/kern/kern_umtx.c:4470 (sleep mutex:umtx lock) 0 0 1 0 4 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:488 (sx:zio->io_lock) 0 0 1 0 4 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:465 (sx:zio->io_lock) 0 0 1 0 4 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:187 (sx:h->hash_mutexes[i]) 0 0 0 0 4 0 0 0 0 /usr/src/sys/vm/vm_glue.c:287 (sleep mutex:vm page) 0 0 0 0 4 0 0 0 0 /usr/src/sys/vm/vm_glue.c:251 (sleep mutex:vm page) 0 0 0 0 4 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:578 (sleep mutex:struct mount mtx) 0 0 0 0 4 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:5312 (sleep mutex:struct mount mtx) 0 0 0 0 4 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:5241 (sleep mutex:struct mount mtx) 0 0 0 0 4 0 0 0 0 /usr/src/sys/kern/subr_taskqueue.c:284 (sleep mutex:mps_taskq) 0 0 0 0 4 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 9) 0 0 0 0 4 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 8) 0 0 0 0 4 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 7) 0 0 0 0 4 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 6) 0 0 0 0 4 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 10) 0 0 0 0 4 0 0 0 0 /usr/src/sys/kern/kern_rangelock.c:164 (sleep mutex:vnode interlock) 0 0 0 0 4 0 0 0 0 /usr/src/sys/kern/kern_prot.c:1233 (sleep mutex:process lock) 0 0 0 0 4 0 0 0 0 /usr/src/sys/kern/kern_exec.c:1050 (sleep mutex:vm page) 0 0 0 0 4 0 0 0 0 /usr/src/sys/kern/kern_exec.c:1028 (sleep mutex:vm page) 0 0 0 0 4 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:489 (sx:zio->io_lock) 0 0 0 0 4 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_rlock.c:538 (sx:zp->z_range_lock) 0 0 0 0 4 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/multilist.c:184 (sx:mls->mls_lock) 0 0 0 0 4 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/multilist.c:148 (sx:mls->mls_lock) 0 0 0 0 4 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dnode.c:1778 (sx:dn->dn_mtx) 0 0 0 0 4 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dnode.c:1173 (sx:dn->dn_mtx) 0 0 0 0 4 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu.c:479 (sx:db->db_mtx) 0 0 0 0 4 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:228 (sx:h->hash_mutexes[i]) 0 0 0 0 4 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:1979 (sx:dn->dn_dbufs_mtx) 0 0 0 0 4 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c:2677 (sx:buf->b_evict_lock) 6 0 12 0 3 4 0 0 0 /usr/src/sys/dev/usb/usb_hub.c:2276 (sleep mutex:ehci1) 3 0 8 0 3 2 0 0 0 /usr/src/sys/dev/usb/usb_hub.c:2276 (sleep mutex:ehci0) 2 0 6 0 3 2 0 0 0 /usr/src/sys/dev/usb/usb_hub.c:2471 (sleep mutex:ehci1) 2 0 4 0 3 1 0 0 1 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 21) 2 0 4 0 3 1 0 0 0 /usr/src/sys/dev/usb/usb_hub.c:2471 (sleep mutex:ehci0) 2 0 4 0 3 1 0 0 0 /usr/src/sys/dev/usb/controller/usb_controller.c:413 (sleep mutex:ehci1) 1 0 2 0 3 0 0 0 0 /usr/src/sys/libkern/arc4random.c:135 (sleep mutex:arc4_mtx) 1 0 2 0 3 0 0 0 0 /usr/src/sys/dev/usb/controller/usb_controller.c:413 (sleep mutex:ehci0) 0 0 2 0 3 0 0 0 1 /usr/src/sys/kern/subr_turnstile.c:895 (spin mutex:sched lock 3) 0 0 2 0 3 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:895 (spin mutex:sched lock 11) 0 0 2 0 3 0 0 0 0 /usr/src/sys/kern/subr_taskqueue.c:467 (spin mutex:mlx4_en_que) 0 0 2 0 3 0 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 1) 0 0 1 0 3 0 0 0 0 /usr/src/sys/kern/kern_synch.c:309 (spin mutex:mlx4_en_que) 0 0 1 0 3 0 0 0 0 /usr/src/sys/dev/usb/controller/usb_controller.c:378 (sleep mutex:ehci1) 0 0 1 0 3 0 0 0 0 /usr/src/sys/dev/usb/controller/usb_controller.c:378 (sleep mutex:ehci0) 0 0 0 0 3 0 0 0 0 /usr/src/sys/vm/uma_core.c:2724 (sleep mutex:sackhole) 0 0 0 0 3 0 0 0 0 /usr/src/sys/vm/uma_core.c:2724 (sleep mutex:32 Bucket) 0 0 0 0 3 0 0 0 0 /usr/src/sys/vm/uma_core.c:2206 (sleep mutex:sackhole) 0 0 0 0 3 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:730 (spin mutex:sched lock 6) 0 0 0 0 3 0 0 0 0 /usr/src/sys/kern/subr_taskqueue.c:284 (spin mutex:mlx4_en_que) 0 0 0 0 3 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 5) 0 0 0 0 3 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 4) 0 0 0 0 3 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 3) 0 0 0 0 3 0 0 0 0 /usr/src/sys/dev/usb/controller/usb_controller.c:691 (sleep mutex:ehci1) 0 0 0 0 3 0 0 0 0 /usr/src/sys/dev/usb/controller/usb_controller.c:691 (sleep mutex:ehci0) 140 0 277 0 2 138 0 0 0 /usr/src/sys/vm/vm_map.c:3315 (sx:vm map (user)) 140 0 277 0 2 138 0 0 0 /usr/src/sys/vm/vm_map.c:3311 (sx:vm map (user)) 103 0 191 0 2 95 0 0 0 /usr/src/sys/kern/kern_fork.c:957 (sx:proctree) 17 0 31 0 2 15 0 0 0 /usr/src/sys/kern/imgact_elf.c:899 (lockmgr:zfs) 14 0 23 0 2 11 0 0 0 /usr/src/sys/kern/kern_exit.c:497 (sleep mutex:process lock) 13 0 24 0 2 12 0 0 0 /usr/src/sys/vm/vm_kern.c:476 (sx:vm map (user)) 12 0 21 0 2 10 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zap_micro.c:515 (sx:zap->zap_rwlock) 11 0 17 0 2 8 0 0 0 /usr/src/sys/kern/kern_exit.c:539 (sleep mutex:process lock) 7 0 11 0 2 5 0 0 0 /usr/src/sys/kern/kern_exec.c:729 (lockmgr:zfs) 5 0 8 0 2 4 0 0 0 /usr/src/sys/kern/kern_exit.c:446 (sx:proctree) 4 102 6 186 2 3 93 0 2 /usr/src/sys/kern/kern_fork.c:958 (sx:allproc) 4 0 7 0 2 3 0 0 0 /usr/src/sys/kern/kern_fork.c:470 (sleep mutex:process lock) 4 0 7 0 2 3 0 0 0 /usr/src/sys/kern/kern_fork.c:469 (sleep mutex:process lock) 4 0 5 0 2 2 0 0 0 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 17) 4 0 5 0 2 2 0 0 0 /usr/src/sys/kern/kern_exec.c:731 (sleep mutex:process lock) 3 0 5 0 2 2 0 0 0 /usr/src/sys/vm/vm_kern.c:445 (sx:vm map (user)) 3 0 5 0 2 2 0 0 0 /usr/src/sys/vm/uma_core.c:1022 (sleep mutex:sackhole) 3 0 5 0 2 2 0 0 0 /usr/src/sys/kern/kern_descrip.c:1855 (sx:filedesc structure) 3 0 5 0 2 2 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/txg.c:566 (sx:tx->tx_sync_lock) 2 338 3 591 2 1 295 0 2 /usr/src/sys/kern/kern_proc.c:711 (sx:proctree) 2 0 5 0 2 2 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/txg.c:520 (sx:tx->tx_sync_lock) 2 0 4 0 2 2 0 0 0 /usr/src/sys/vm/vm_object.c:495 (lockmgr:zfs) 2 0 4 0 2 2 0 0 0 /usr/src/sys/kern/kern_exit.c:594 (spin mutex:process slock) 2 0 3 0 2 1 0 0 0 /usr/src/sys/kern/subr_taskqueue.c:467 (spin mutex:mlx4_en_que) 2 0 2 0 2 1 0 0 0 /usr/src/sys/kern/kern_sig.c:968 (sleep mutex:sigacts) 2 0 2 0 2 1 0 0 0 /usr/src/sys/kern/kern_exit.c:1160 (sleep mutex:process lock) 1 2 1 2 2 0 1 0 1 /usr/src/sys/kern/subr_turnstile.c:210 (spin mutex:sched lock 18) 1 0 3 0 2 1 0 0 0 /usr/src/sys/kern/kern_exit.c:216 (sleep mutex:process lock) 1 0 2 0 2 1 0 0 0 /usr/src/sys/vm/vm_object.c:497 (rw:vm object) 1 0 2 0 2 1 0 0 0 /usr/src/sys/kern/kern_thread.c:533 (spin mutex:pstatl) 1 0 2 0 2 1 0 0 0 /usr/src/sys/kern/kern_synch.c:309 (spin mutex:mlx4_en_que) 1 0 2 0 2 1 0 0 0 /usr/src/sys/kern/kern_fork.c:589 (sleep mutex:process lock) 1 0 2 0 2 1 0 0 0 /usr/src/sys/kern/kern_fork.c:587 (sx:proctree) 1 0 2 0 2 1 0 0 0 /usr/src/sys/kern/kern_fork.c:404 (sleep mutex:process lock) 1 0 2 0 2 1 0 0 0 /usr/src/sys/kern/kern_exit.c:1161 (spin mutex:process slock) 1 0 2 0 2 1 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zap.c:523 (sx:l->l_rwlock) 1 0 1 0 2 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:2223 (spin mutex:sched lock 12) 1 0 1 0 2 0 0 0 0 /usr/src/sys/kern/subr_taskqueue.c:467 (sleep mutex:mps_taskq) 1 0 1 0 2 0 0 0 0 /usr/src/sys/kern/kern_proc.c:576 (sleep mutex:process group) 1 0 1 0 2 0 0 0 0 /usr/src/sys/kern/kern_fork.c:702 (sleep mutex:process lock) 1 0 1 0 2 0 0 0 0 /usr/src/sys/kern/kern_fork.c:701 (sleep mutex:process lock) 1 0 1 0 2 0 0 0 0 /usr/src/sys/kern/kern_descrip.c:2176 (sleep mutex:process lock) 1 0 1 0 2 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_errlog.c:349 (sx:spa->spa_errlist_lock) 0 1 0 2 2 0 1 0 2 /usr/src/sys/kern/subr_turnstile.c:210 (spin mutex:sched lock 3) 0 0 1 0 2 0 0 0 1 /usr/src/sys/kern/subr_turnstile.c:895 (spin mutex:sched lock 9) 0 0 1 0 2 0 0 0 0 /usr/src/sys/vm/vm_map.c:3471 (sx:vm map (user)) 0 0 1 0 2 0 0 0 0 /usr/src/sys/vm/uma_core.c:2371 (sleep mutex:32 Bucket) 0 0 1 0 2 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:2742 (sleep mutex:vnode interlock) 0 0 1 0 2 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:2223 (spin mutex:sched lock 14) 0 0 1 0 2 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:895 (spin mutex:sched lock 7) 0 0 1 0 2 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:895 (spin mutex:sched lock 2) 0 0 1 0 2 0 0 0 0 /usr/src/sys/kern/subr_taskqueue.c:467 (spin mutex:mlx4_en_que) 0 0 1 0 2 0 0 0 0 /usr/src/sys/kern/subr_eventhandler.c:254 (sleep mutex:process_exec) 0 0 1 0 2 0 0 0 0 /usr/src/sys/kern/subr_eventhandler.c:254 (sleep mutex:process_dtor) 0 0 1 0 2 0 0 0 0 /usr/src/sys/kern/kern_synch.c:309 (spin mutex:mlx4_en_que) 0 0 1 0 2 0 0 0 0 /usr/src/sys/kern/kern_sig.c:3632 (sleep mutex:sigacts) 0 0 1 0 2 0 0 0 0 /usr/src/sys/kern/kern_proc.c:552 (sleep mutex:process group) 0 0 1 0 2 0 0 0 0 /usr/src/sys/kern/kern_proc.c:480 (sleep mutex:process group) 0 0 1 0 2 0 0 0 0 /usr/src/sys/kern/kern_mutex.c:190 (sleep mutex:mps_taskq) 0 0 1 0 2 0 0 0 0 /usr/src/sys/kern/kern_ktrace.c:575 (sleep mutex:process lock) 0 0 1 0 2 0 0 0 0 /usr/src/sys/kern/kern_fork.c:590 (sleep mutex:process lock) 0 0 1 0 2 0 0 0 0 /usr/src/sys/kern/kern_fork.c:588 (sleep mutex:process group) 0 0 1 0 2 0 0 0 0 /usr/src/sys/kern/kern_exit.c:843 (sleep mutex:process lock) 0 0 1 0 2 0 0 0 0 /usr/src/sys/kern/kern_exit.c:796 (sleep mutex:process lock) 0 0 1 0 2 0 0 0 0 /usr/src/sys/kern/kern_descrip.c:2305 (sx:filedesc structure) 0 0 1 0 2 0 0 0 0 /usr/src/sys/kern/imgact_elf.c:972 (sleep mutex:process lock) 0 0 1 0 2 0 0 0 0 /usr/src/sys/kern/imgact_elf.c:1034 (lockmgr:zfs) 0 0 0 0 2 0 0 0 1 /usr/src/sys/kern/subr_turnstile.c:210 (spin mutex:sched lock 7) 0 0 0 0 2 0 0 0 0 /usr/src/sys/vm/vm_object.c:1774 (rw:vm object) 0 0 0 0 2 0 0 0 0 /usr/src/sys/vm/vm_map.c:3985 (sleep mutex:vm map (system)) 0 0 0 0 2 0 0 0 0 /usr/src/sys/vm/vm_map.c:3368 (rw:vm object) 0 0 0 0 2 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:mbuf) 0 0 0 0 2 0 0 0 0 /usr/src/sys/vm/uma_core.c:2724 (sleep mutex:16384) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/vfs_vnops.c:2468 (sx:pmc-sx) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:2823 (sleep mutex:vnode_free_list) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:2757 (sleep mutex:vnode_free_list) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/sysv_sem.c:1399 (sleep mutex:semu) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:730 (spin mutex:sched lock 8) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:730 (spin mutex:sched lock 5) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:210 (spin mutex:sched lock 11) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/subr_taskqueue.c:284 (spin mutex:mlx4_en_que) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/subr_taskqueue.c:284 (spin mutex:mlx4_en_que) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 0) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/subr_eventhandler.c:254 (sleep mutex:process_exit) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/subr_eventhandler.c:254 (sleep mutex:process_ctor) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_umtx.c:4477 (spin mutex:sched lock 19) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_umtx.c:4350 (sleep mutex:process lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_umtx.c:4340 (sleep mutex:process lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_thread.c:1200 (rw:tidhash) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_thread.c:1192 (rw:tidhash) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_sig.c:2136 (sleep mutex:sigacts) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_proc.c:775 (sleep mutex:process lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_proc.c:702 (sleep mutex:process lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_proc.c:615 (sleep mutex:ttymtx) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_proc.c:607 (sleep mutex:process group) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_proc.c:577 (sleep mutex:process lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_proc.c:554 (sleep mutex:process lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_proc.c:553 (sleep mutex:process group) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_proc.c:382 (sx:allproc) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_proc.c:224 (sleep mutex:process_dtor) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_proc.c:197 (sleep mutex:process_ctor) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_proc.c:1922 (sleep mutex:process lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_ktrace.c:576 (sleep mutex:ktrace) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_jail.c:2754 (sleep mutex:jail mutex) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_jail.c:2743 (sleep mutex:jail mutex) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_fork.c:774 (sleep mutex:process lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_fork.c:704 (spin mutex:process slock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_fork.c:598 (sleep mutex:session) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_fork.c:405 (sleep mutex:process lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_exit.c:856 (sleep mutex:process lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_exit.c:835 (sleep mutex:process lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_exit.c:829 (sx:allproc) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_exit.c:601 (spin mutex:pstatl) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_exit.c:540 (sleep mutex:sigacts) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_exit.c:425 (sx:allproc) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_exit.c:338 (sleep mutex:process lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_exit.c:1275 (sleep mutex:process lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_exit.c:1055 (spin mutex:process slock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_exec.c:907 (sleep mutex:process lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_exec.c:394 (sleep mutex:process lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_event.c:2133 (sleep mutex:process lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_descrip.c:2183 (sx:filedesc structure) 0 0 0 0 2 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c:6273 (sx:spa->spa_async_lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c:6260 (sx:spa->spa_async_lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/rrwlock.c:216 (sx:rrl->rr_lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/bplist.c:67 (sx:bpl->bpl_lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/amd64/amd64/vm_machdep.c:305 (sleep mutex:descriptor tables) 0 0 0 0 2 0 0 0 0 /usr/src/sys/amd64/amd64/vm_machdep.c:250 (sleep mutex:descriptor tables) 0 0 0 0 2 0 0 0 0 /usr/src/sys/amd64/amd64/machdep.c:592 (sleep mutex:descriptor tables) 0 0 0 0 2 0 0 0 0 /usr/src/sys/amd64/amd64/../../kern/subr_syscall.c:202 (sleep mutex:process lock) 179 0 179 0 1 179 0 0 0 /usr/src/sys/vm/uma_core.c:1747 (rw:UMA lock) 25 0 25 0 1 25 0 0 0 /usr/src/sys/vm/vm_glue.c:754 (sx:allproc) 3 0 3 0 1 3 0 0 0 /usr/src/sys/vm/uma_core.c:2371 (sleep mutex:16) 2 0 2 0 1 2 0 0 0 /usr/src/sys/kern/subr_vmem.c:732 (sleep mutex:vmem list lock) 2 0 2 0 1 2 0 0 0 /usr/src/sys/kern/subr_turnstile.c:895 (spin mutex:sched lock 22) 2 0 2 0 1 2 0 0 0 /usr/src/sys/kern/subr_turnstile.c:895 (spin mutex:sched lock 12) 2 0 2 0 1 2 0 0 0 /usr/src/sys/kern/kern_thread.c:534 (spin mutex:sched lock 19) 1 2 1 2 1 1 2 0 1 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 17) 1 0 1 0 1 1 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_5242880) 1 0 1 0 1 1 0 0 0 /usr/src/sys/vm/uma_core.c:2371 (sleep mutex:sackhole) 1 0 1 0 1 1 0 0 0 /usr/src/sys/vm/uma_core.c:1022 (sleep mutex:16) 1 0 1 0 1 1 0 0 0 /usr/src/sys/kern/subr_turnstile.c:895 (spin mutex:sched lock 23) 1 0 1 0 1 1 0 0 0 /usr/src/sys/kern/subr_turnstile.c:895 (spin mutex:sched lock 20) 1 0 1 0 1 1 0 0 0 /usr/src/sys/kern/subr_turnstile.c:895 (spin mutex:sched lock 18) 1 0 1 0 1 1 0 0 0 /usr/src/sys/kern/subr_turnstile.c:895 (spin mutex:sched lock 16) 1 0 1 0 1 1 0 0 0 /usr/src/sys/kern/subr_trap.c:270 (spin mutex:sched lock 23) 1 0 1 0 1 1 0 0 0 /usr/src/sys/kern/subr_taskqueue.c:467 (spin mutex:mlx4_en_que) 1 0 1 0 1 1 0 0 0 /usr/src/sys/kern/subr_taskqueue.c:467 (spin mutex:mlx4_en_que) 1 0 1 0 1 1 0 0 0 /usr/src/sys/kern/subr_taskqueue.c:467 (sleep mutex:thread) 1 0 1 0 1 1 0 0 0 /usr/src/sys/kern/kern_thread.c:534 (spin mutex:sched lock 23) 1 0 1 0 1 1 0 0 0 /usr/src/sys/kern/kern_fork.c:764 (spin mutex:sched lock 20) 1 0 1 0 1 1 0 0 0 /usr/src/sys/kern/kern_fork.c:764 (spin mutex:sched lock 2) 1 0 1 0 1 1 0 0 0 /usr/src/sys/kern/kern_fork.c:491 (spin mutex:sched lock 20) 0 3 0 3 1 0 3 0 1 /usr/src/sys/kern/subr_turnstile.c:210 (spin mutex:sched lock 12) 0 2 0 2 1 0 2 0 1 /usr/src/sys/kern/sched_ule.c:2175 (spin mutex:sched lock 2) 0 0 0 0 1 0 0 0 1 /usr/src/sys/kern/subr_turnstile.c:210 (spin mutex:sched lock 8) 0 0 0 0 1 0 0 0 1 /usr/src/sys/kern/subr_turnstile.c:210 (spin mutex:sched lock 23) 0 0 0 0 1 0 0 0 1 /usr/src/sys/kern/subr_turnstile.c:210 (spin mutex:sched lock 2) 0 0 0 0 1 0 0 0 1 /usr/src/sys/kern/subr_turnstile.c:210 (spin mutex:sched lock 16) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_link_cache) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_98304) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_917504) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_8388608) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_81920) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_8192) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_786432) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_7340032) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_7168) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_655360) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_65536) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_6291456) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_6144) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_57344) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_524288) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_5120) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_512) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_49152) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_458752) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_4194304) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_40960) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_4096) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_393216) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_3670016) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_3584) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_327680) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_32768) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_3145728) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_3072) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_28672) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_2621440) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_262144) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_2560) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_24576) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_229376) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_2097152) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_20480) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_2048) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_196608) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_1835008) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_16777216) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_163840) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_16384) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_1572864) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_1536) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_14680064) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_14336) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_1310720) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_131072) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_12582912) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_12288) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_114688) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_10485760) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_1048576) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_10240) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_1024) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_cache) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_98304) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_917504) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_8388608) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_81920) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_8192) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_786432) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_7340032) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_7168) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_655360) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_65536) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_6291456) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_6144) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_57344) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_5242880) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_524288) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_5120) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_512) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_49152) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_458752) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_4194304) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_40960) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_4096) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_393216) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_3670016) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_3584) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_327680) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_32768) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_3145728) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_3072) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_28672) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_2621440) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_262144) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_2560) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_24576) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_229376) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_2097152) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_20480) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_2048) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_196608) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_1835008) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_16777216) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_163840) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_16384) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_1572864) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_1536) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_14680064) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_14336) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_1310720) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_131072) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_12582912) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_12288) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_114688) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_10485760) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_1048576) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_10240) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_1024) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zil_lwb_cache) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zfs_znode_cache) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:vtnet_tx_hdr) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:vmem btag) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:unpcb) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:umtx_shm) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:umtx pi) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:udplite_inpcb) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:udpcb) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:udp_inpcb) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:ttyoutq) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:ttyinq) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:tcptw) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:tcpreass) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:tcpcb) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:tcp_inpcb) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:taskq_zone) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:syncache) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:socket) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:selfd) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:sctp_stream_msg_out) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:sctp_readq) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:sctp_raddr) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:sctp_laddr) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:sctp_ep) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:sctp_chunk) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:sctp_asoc) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:sctp_asconf_ack) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:sctp_asconf) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:sackhole) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:sa_cache) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:rtentry) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:rl_entry) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:ripcb) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:range_seg_cache) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:procdesc) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:pipe) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:nvme_request) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:mt_zone) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:mlx4_cmd) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:mbuf_jumbo_page) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:mbuf_jumbo_9k) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:mbuf_jumbo_16k) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:mbuf_cluster) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:lz4_ctx) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:ksiginfo) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:itimer) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:ipq) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:hostcache) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:g_bio) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:filedesc0) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:fakepg) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:dnode_t) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:dmu_buf_impl_t) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:cryptop) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:cryptodesc) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:cpuset) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:audit_record) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:arc_buf_t) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:arc_buf_hdr_t_l2only) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:arc_buf_hdr_t_full) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:VNODEPOLL) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:VNODE) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:VMSPACE) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:VM OBJECT) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:UMA Zones) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:UMA Slabs) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:UMA Kegs) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:UMA Hash) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:TURNSTILE) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:THREAD) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:SWAPMETA) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:STS VFS Cache) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:SLEEPQUEUE) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:S VFS Cache) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:RADIX NODE) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:PROC) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:NCLNODE) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:NAMEI) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:Mountpoints) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:MAP) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:MAP ENTRY) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:MAC labels) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:LTS VFS Cache) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:L VFS Cache) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:KNOTE) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:KMAP ENTRY) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:Files) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:FPU_save_area) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:DMAR_MAP_ENTRY) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:DIRHASH) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:BUF TRIE) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:AIOP) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:AIOLIO) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:AIOL) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:AIOCB) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:AIO) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:8192) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:8 Bucket) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:65536) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:64) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:64 pcpu) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:64 Bucket) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:6 Bucket) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:512) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:4096) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:4 Bucket) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:32768) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:32) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:32 Bucket) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:256) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:256 Bucket) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:2048) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:16384) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:16) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:16 Bucket) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:128) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:128 Bucket) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:12 Bucket) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:1024) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:2724 (sleep mutex:4096) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:2259 (sleep mutex:sackhole) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:2259 (sleep mutex:16) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:2206 (sleep mutex:zio_cache) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:2206 (sleep mutex:RADIX NODE) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:2206 (sleep mutex:16384) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:2206 (sleep mutex:16) 0 0 0 0 1 0 0 0 0 /usr/src/sys/modules/mlxen/../../ofed/drivers/net/mlx4/en_rx.c:724 (spin mutex:sched lock 7) 0 0 0 0 1 0 0 0 0 /usr/src/sys/modules/mlxen/../../ofed/drivers/net/mlx4/en_rx.c:724 (spin mutex:sched lock 22) 0 0 0 0 1 0 0 0 0 /usr/src/sys/modules/mlxen/../../ofed/drivers/net/mlx4/en_rx.c:724 (spin mutex:sched lock 16) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:2223 (spin mutex:sched lock 9) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:2223 (spin mutex:sched lock 8) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:2223 (spin mutex:sched lock 21) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:2223 (spin mutex:sched lock 20) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:2223 (spin mutex:sched lock 18) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:2223 (spin mutex:sched lock 15) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:895 (spin mutex:sched lock 6) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:895 (spin mutex:sched lock 5) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:730 (spin mutex:sched lock 9) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:730 (spin mutex:sched lock 7) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:730 (spin mutex:sched lock 11) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:730 (spin mutex:sched lock 10) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:210 (spin mutex:sched lock 9) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:210 (spin mutex:sched lock 20) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/subr_trap.c:232 (spin mutex:sched lock 23) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/subr_taskqueue.c:467 (spin mutex:mlx4_en_que) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/subr_taskqueue.c:284 (spin mutex:mlx4_en_que) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/subr_taskqueue.c:284 (spin mutex:mlx4_en_que) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/subr_taskqueue.c:284 (spin mutex:mlx4_en_que) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/subr_taskqueue.c:284 (sleep mutex:thread) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 19) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2754 (spin mutex:sched lock 8) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2754 (spin mutex:sched lock 19) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 1) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2175 (spin mutex:sched lock 20) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/kern_umtx.c:4477 (spin mutex:sched lock 8) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/kern_umtx.c:4477 (spin mutex:sched lock 23) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/kern_thread.c:576 (spin mutex:sched lock 23) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/kern_thread.c:576 (spin mutex:sched lock 19) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/kern_synch.c:309 (spin mutex:mlx4_en_que) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/kern_synch.c:309 (spin mutex:mlx4_en_que) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/kern_synch.c:309 (spin mutex:mlx4_en_que) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/kern_rmlock.c:415 (sleep mutex:in_ifaddr_lock) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/kern_proc.c:1030 (spin mutex:sched lock 23) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/kern_proc.c:1030 (spin mutex:sched lock 20) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/kern_proc.c:1030 (spin mutex:sched lock 2) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/kern_mutex.c:190 (sleep mutex:thread) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/kern_fork.c:491 (spin mutex:sched lock 2) srv2 max wait_max total wait_total count avg wait_avg cnt_hold cnt_lock name 13 15 2486474 77023 4268362 0 0 0 48052 /usr/src/sys/netinet/tcp_output.c:328 (sleep mutex:so_snd) 5 6 537175 51158 2981766 0 0 0 210559 /usr/src/sys/kern/kern_timeout.c:578 (spin mutex:callout) 11 15 672731 42907 2945239 0 0 0 27773 /usr/src/sys/kern/uipc_socket.c:1756 (sleep mutex:so_rcv) 11 8 2475905 42239 2600341 0 0 0 52946 /usr/src/sys/kern/subr_sleepqueue.c:254 (spin mutex:sleepq chain) 2 4 110041 6293 1861473 0 0 0 37606 /usr/src/sys/dev/random/random_harvestq.c:419 (spin mutex:entropy harvest mutex) 108 39 10969200 148402 1859731 5 0 0 70205 /usr/src/sys/netinet/in_pcb.c:1964 (rw:tcpinp) 7 0 464969 0 1859731 0 0 0 0 /usr/src/sys/netinet/in_pcb.c:1957 (rw:pcbinfohash) 11 12 387173 28471 1819598 0 0 0 19230 /usr/src/sys/kern/uipc_socket.c:1529 (sleep mutex:so_rcv) 16 0 1003556 0 1727541 0 0 0 0 /usr/src/sys/modules/mlxen/../../ofed/drivers/net/mlx4/en_tx.c:1084 (sleep mutex:mlx4 tx) 8 0 225751 0 1726427 0 0 0 0 /usr/src/sys/modules/mlx4/../../ofed/drivers/net/mlx4/cq.c:59 (rw:lnxrw) 0 0 102517 0 1383677 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 0) 19 50 197776 16087 1244808 0 0 0 5318 /usr/src/sys/kern/uipc_socket.c:1258 (sleep mutex:so_snd) 11 15 1298587 95302 1019801 1 0 0 50270 /usr/src/sys/netinet/tcp_input.c:1908 (sleep mutex:so_rcv) 42 102 504586 245927 954528 0 0 0 59884 /usr/src/sys/netinet/tcp_usrreq.c:814 (rw:tcpinp) 3 0 47145 0 954528 0 0 0 0 /usr/src/sys/kern/uipc_socket.c:1919 (sleep mutex:so_rcv) 549894 0 157159244 0 954516 164 0 0 0 /usr/src/sys/kern/uipc_sockbuf.c:283 (sx:so_rcv_sx) 12 14 55647 43861 865070 0 0 0 21571 /usr/src/sys/kern/kern_mutex.c:190 (sleep mutex:so_rcv) 17 246 742654 2335533 816692 0 2 0 249721 /usr/src/sys/netinet/tcp_usrreq.c:874 (rw:tcpinp) 6 0 247768 0 816692 0 0 0 0 /usr/src/sys/kern/uipc_sockbuf.c:682 (sleep mutex:so_snd) 1 6 180808 119 699280 0 0 0 130 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 0) 1 7 180296 49 692477 0 0 0 25 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 0) 0 3 59788 18 691598 0 0 0 17 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 0) 24 11 611902 1193 519348 1 0 0 923 /usr/src/sys/netinet/tcp_input.c:2826 (sleep mutex:so_snd) 8 48 30534 1140823 428116 0 2 0 339152 /usr/src/sys/kern/kern_mutex.c:190 (sleep mutex:so_snd) 0 0 24904 0 252526 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 10) 2 0 60676 0 249181 0 0 0 0 /usr/src/sys/kern/kern_clocksource.c:216 (spin mutex:et_hw_mtx) 0 0 24332 0 245231 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 8) 0 0 24963 0 240677 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 11) 2 0 23220 0 234134 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 9) 0 0 22996 0 233973 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 6) 2 0 15990 0 226409 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 12) 0 0 22556 0 226114 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 7) 0 0 22341 0 224864 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 4) 1 0 14326 0 219662 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 14) 2 0 14621 0 219140 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 20) 0 0 21778 0 217041 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 5) 0 0 21604 0 216047 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 2) 0 0 20801 0 209320 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 3) 3 0 13882 0 208838 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 22) 2 0 13431 0 206551 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 19) 1 1 19509 43 203701 0 0 0 89 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 0) 1133859 0 157101043 0 200705 782 0 0 0 /usr/src/sys/kern/uipc_sockbuf.c:283 (sx:so_snd_sx) 1 0 12559 0 195413 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 21) 2 0 12496 0 194255 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 18) 8 7 76577 583 190691 0 0 0 442 /usr/src/sys/kern/uipc_sockbuf.c:1148 (sleep mutex:so_snd) 8 6 169348 181 190691 0 0 0 163 /usr/src/sys/netinet/tcp_input.c:1808 (sleep mutex:so_snd) 1 0 12201 0 190610 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 16) 2 0 12359 0 174152 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 13) 2 0 11878 0 174023 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 23) 2 0 11543 0 169516 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 17) 1 0 10422 0 161420 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 15) 0 1 21741 336 125710 0 0 0 788 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 10) 0 4 20379 298 122018 0 0 0 725 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 8) 1 1 26973 390 119775 0 0 0 899 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 11) 1 1 20669 297 116487 0 0 0 833 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 9) 1 3 20296 361 116427 0 0 0 907 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 6) 4 1 29876 43 114662 0 0 0 139 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 10) 2 4 31622 40 113162 0 0 0 50 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 12) 2 0 15373 0 113111 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 12) 0 0 6279 0 113111 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 12) 1 0 7046 0 112961 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 12) 2 2 57984 2097 112860 0 0 0 7786 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 10) 0 0 36081 57 112858 0 0 0 138 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 10) 0 1 19861 293 112474 0 0 0 804 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 7) 0 1 19270 325 111851 0 0 0 833 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 4) 3 1 29358 43 111793 0 0 0 144 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 8) 2 2 29943 13 109801 0 0 0 18 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 14) 2 1 14951 1 109740 0 0 0 1 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 14) 0 0 6063 0 109740 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 14) 2 1 28713 38 109700 0 0 0 120 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 11) 1 0 6516 0 109554 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 14) 2 5 29798 26 109520 0 0 0 19 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 20) 2 0 15342 0 109469 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 20) 0 0 5962 0 109469 0 0 0 1 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 20) 2 0 6771 0 109314 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 20) 1 3 56198 1848 108740 0 0 0 7399 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 8) 0 0 34338 65 108740 0 0 0 155 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 8) 6 3 61499 386 108321 0 0 0 885 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 12) 1 3 55735 2153 108268 0 0 0 7947 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 11) 1 1 34652 62 108268 0 0 0 155 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 11) 77 0 890397 0 108172 8 0 0 0 /usr/src/sys/modules/mlxen/../../ofed/drivers/net/mlx4/en_tx.c:511 (sleep mutex:mlx4 comp) 0 2 20209 267 107954 0 0 0 708 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 5) 0 1 18629 291 107465 0 0 0 658 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 2) 2 2 27944 44 107055 0 0 0 142 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 6) 2 1 28082 44 106823 0 0 0 134 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 9) 5 2 58990 458 105735 0 0 0 1373 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 14) 1 3 55036 1763 105304 0 0 0 6871 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 9) 1 2 33254 57 105302 0 0 0 138 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 9) 4 2 58969 389 104602 0 0 0 880 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 20) 2 4 28378 38 104375 0 0 0 27 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 22) 2 0 14149 0 104301 0 0 0 1 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 22) 1 0 5786 0 104301 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 22) 0 2 17863 265 104101 0 0 0 600 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 3) 1 0 6510 0 103953 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 22) 1 1 27187 36 103882 0 0 0 129 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 7) 1 3 53339 1797 103729 0 0 0 6843 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 6) 1 0 32882 66 103729 0 0 0 146 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 6) 2 2 27771 66 103406 0 0 0 194 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 4) 2 3 28575 19 103254 0 0 0 21 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 19) 3 0 14001 0 103177 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 19) 0 0 5810 0 103177 0 0 0 1 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 19) 0 0 6225 0 103032 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 19) 1 3 51231 1739 100136 0 0 0 6646 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 7) 4 1 58406 378 100134 0 0 0 854 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 22) 1 0 31690 50 100131 0 0 0 116 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 7) 3 1 26523 51 100110 0 0 0 164 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 5) 1 1 26109 60 99778 0 0 0 202 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 2) 1 3 51495 1832 99243 0 0 0 6822 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 4) 1 1 32204 55 99240 0 0 0 121 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 4) 4 2 54304 422 99072 0 0 0 941 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 19) 2 3 27299 23 97668 0 0 0 23 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 21) 1 0 13335 0 97588 0 0 0 1 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 21) 1 0 5573 0 97588 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 21) 1 0 5893 0 97475 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 21) 2 0 25218 39 97029 0 0 0 162 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 3) 3 3 27014 25 96895 0 0 0 17 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 18) 1 0 12966 0 96780 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 18) 0 0 5468 0 96780 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 18) 1 0 5840 0 96596 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 18) 1 3 49220 1634 95735 0 0 0 6725 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 5) 0 1 30593 52 95732 0 0 0 129 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 5) 2 2 27223 20 95268 0 0 0 25 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 16) 4 13 21880 13 95232 0 0 0 1 /usr/src/sys/dev/random/fortuna.c:222 (sleep mutex:reseed mutex) 4 0 13052 0 95194 0 0 0 1 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 16) 0 0 5398 0 95194 0 0 0 1 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 16) 0 0 5789 0 95126 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 16) 1 2 48364 1611 94639 0 0 0 6226 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 2) 0 0 30103 44 94637 0 0 0 105 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 2) 4 2 54663 338 93219 0 0 0 769 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 21) 5 2 53772 405 92653 0 0 0 967 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 18) 0 1 29402 41 91308 0 0 0 97 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 3) 2 2 47156 1500 91307 0 0 0 6036 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 3) 8 2 52455 486 90406 0 0 0 1352 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 16) 2 2 24572 18 87042 0 0 0 18 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 13) 2 3 24381 25 86989 0 0 0 25 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 23) 2 0 11924 0 86972 0 0 0 1 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 13) 0 1 4931 1 86972 0 0 0 1 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 13) 1 0 12126 0 86894 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 23) 0 0 4967 0 86894 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 23) 1 0 5232 0 86656 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 13) 1 0 5544 0 86581 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 23) 2 2 23758 17 84722 0 0 0 24 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 17) 2 0 11403 0 84655 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 17) 1 0 4763 0 84655 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 17) 1 0 5349 0 84582 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 17) 5 2 46200 343 82797 0 0 0 800 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 13) 4 2 46295 278 82400 0 0 0 642 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 23) 2 3 23021 30 80669 0 0 0 21 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 15) 1 0 10748 0 80610 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 15) 0 1 4564 1 80610 0 0 0 1 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 15) 1 0 5107 0 80428 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 15) 6 2 49217 330 80222 0 0 0 719 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 17) 4 2 44226 372 76625 0 0 0 966 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 15) 1 0 9978 0 59667 0 0 0 0 /usr/src/sys/kern/kern_clocksource.c:760 (spin mutex:et_hw_mtx) 12 0 3870 0 32979 0 0 0 0 /usr/src/sys/vm/uma_core.c:2724 (sleep mutex:mbuf) 3 0 6129 0 32969 0 0 0 0 /usr/src/sys/vm/uma_core.c:2206 (sleep mutex:mbuf) 29 10 45970 1682 24566 1 0 0 870 /usr/src/sys/netinet/tcp_reass.c:316 (sleep mutex:so_rcv) 2 0 3956 0 23705 0 0 0 0 /usr/src/sys/vm/uma_core.c:2206 (sleep mutex:mbuf_jumbo_page) 4 0 3155 0 23696 0 0 0 0 /usr/src/sys/vm/uma_core.c:2724 (sleep mutex:mbuf_jumbo_page) 0 1 1886 9 13877 0 0 0 14 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 8) 0 0 852 7 13877 0 0 0 15 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 8) 0 0 965 0 13685 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 8) 0 1 1853 8 13403 0 0 0 12 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 10) 0 0 828 10 13403 0 0 0 23 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 10) 0 0 1825 7 13387 0 0 0 12 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 2) 0 0 821 5 13386 0 0 0 11 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 2) 0 1 1808 8 13354 0 0 0 11 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 3) 0 0 825 5 13354 0 0 0 13 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 3) 0 0 1691 8 13259 0 0 0 13 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 6) 0 0 836 6 13256 0 0 0 12 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 6) 0 0 935 0 13228 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 10) 0 0 923 0 13200 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 2) 0 1 1806 3 13193 0 0 0 4 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 4) 0 0 816 4 13188 0 0 0 9 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 4) 0 0 915 0 13169 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 3) 0 0 938 0 13081 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 6) 0 0 894 0 13025 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 4) 0 0 1740 5 12928 0 0 0 7 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 7) 0 0 809 6 12927 0 0 0 16 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 7) 0 1 795 7 12789 0 0 0 14 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 5) 0 0 1737 4 12789 0 0 0 6 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 5) 0 0 906 0 12710 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 7) 0 0 891 0 12647 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 5) 0 0 1652 3 12070 0 0 0 4 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 11) 0 1 790 6 12069 0 0 0 13 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 11) 0 0 844 0 11914 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 11) 0 1 1525 5 11764 0 0 0 8 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 9) 0 0 731 6 11763 0 0 0 14 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 9) 0 0 802 0 11634 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 9) 5 4 1172 77 11368 0 0 0 60 /usr/src/sys/netinet/tcp_input.c:2678 (sleep mutex:so_snd) 4 0 1928 0 10746 0 0 0 0 /usr/src/sys/vm/uma_core.c:2206 (sleep mutex:mbuf_jumbo_9k) 2 0 1182 0 10744 0 0 0 0 /usr/src/sys/vm/uma_core.c:2724 (sleep mutex:mbuf_jumbo_9k) 1 0 766 0 6035 0 0 0 0 /usr/src/sys/kern/kern_clocksource.c:827 (spin mutex:et_hw_mtx) 1 2 698 3 5284 0 0 0 4 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sleepq chain) 0 1 531 4 5201 0 0 0 5 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sleepq chain) 3 1 3589 3 4894 0 0 0 3 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 20) 5 0 3680 0 4841 0 0 0 0 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 16) 1 0 900 1 4821 0 0 0 3 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 20) 3 1 3627 1 4815 0 0 0 1 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 12) 1 2 906 5 4787 0 0 0 5 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 16) 1 1 958 2 4782 0 0 0 3 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 12) 6 0 3314 0 4564 0 0 0 0 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 23) 2 1 853 1 4511 0 0 0 1 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 23) 3 1 3295 3 4484 0 0 0 2 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 17) 2 0 916 0 4434 0 0 0 1 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 17) 3 0 3235 0 4431 0 0 0 0 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 21) 4 0 3090 1 4424 0 0 0 2 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 18) 2 1 802 3 4406 0 0 0 3 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 21) 6 0 3105 0 4229 0 0 0 0 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 13) 9 3 3116 3 4214 0 0 0 1 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 22) 4 3 816 4 4185 0 0 0 3 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 22) 3 0 827 1 4177 0 0 0 2 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 13) 3 3 2981 3 4164 0 0 0 1 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 19) 5 1 2863 66 4155 0 0 0 192 /usr/src/sys/kern/kern_timeout.c:751 (spin mutex:callout) 5 0 3300 6 4145 0 0 0 27 /usr/src/sys/kern/kern_timeout.c:449 (spin mutex:callout) 2 0 809 0 4124 0 0 0 1 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 19) 3 3 797 3 4120 0 0 0 1 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 18) 4 0 2997 0 4049 0 0 0 0 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 14) 5 0 2921 0 4024 0 0 0 0 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 15) 2 3 713 5 3978 0 0 0 2 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 15) 1 2 769 5 3974 0 0 0 4 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 14) 9 6 2537 173 3264 0 0 0 79 /usr/src/sys/modules/mlxen/../../ofed/drivers/net/mlx4/en_tx.c:1055 (sleep mutex:mlx4 tx) 4 0 655 0 2943 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:560 (spin mutex:turnstile lock) 2 0 1391 0 2943 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:551 (spin mutex:turnstile chain) 7 0 4709 0 2108 2 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:921 (spin mutex:sleepq chain) 1 0 309 0 2098 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1976 (spin mutex:sched lock 1) 3 0 210 0 1898 0 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:4367 (sleep mutex:pmap) 1 1 564 38 1646 0 0 0 130 /usr/src/sys/kern/kern_timeout.c:864 (spin mutex:callout) 0 0 189 3 1320 0 0 0 7 /usr/src/sys/kern/subr_taskqueue.c:284 (spin mutex:mlx4_en_que) 3 10 1006 747 1319 0 0 0 743 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 0) 2 0 583 1 1319 0 0 0 5 /usr/src/sys/kern/subr_taskqueue.c:467 (spin mutex:mlx4_en_que) 0 0 53 0 1308 0 0 0 0 /usr/src/sys/cam/cam_xpt.c:4775 (sleep mutex:CAM bus lock) 0 0 53 0 1308 0 0 0 0 /usr/src/sys/cam/cam_xpt.c:4660 (sleep mutex:CAM bus lock) 1 0 86 0 1280 0 0 0 0 /usr/src/sys/modules/mlxen/../../ofed/drivers/net/mlx4/en_netdev.c:1097 (sleep mutex:lnxspin) 0 0 57 0 1280 0 0 0 0 /usr/src/sys/netinet/ip_reass.c:491 (sleep mutex:IP reassembly) 1 0 321 0 1277 0 0 0 0 /usr/src/sys/kern/kern_synch.c:309 (spin mutex:mlx4_en_que) 0 0 245 3 1270 0 0 0 10 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 9) 0 0 241 4 1270 0 0 0 13 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 6) 0 0 188 0 1270 0 0 0 0 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 0) 0 1 240 5 1269 0 0 0 12 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 8) 0 0 245 2 1269 0 0 0 6 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 3) 0 0 243 3 1269 0 0 0 6 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 11) 0 0 236 2 1269 0 0 0 7 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 7) 0 1 256 4 1268 0 0 0 11 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 2) 0 1 243 7 1268 0 0 0 12 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 4) 0 0 244 3 1268 0 0 0 9 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 10) 0 0 241 1 1268 0 0 0 4 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 5) 7 0 368 0 1229 0 0 0 0 /usr/src/sys/vm/uma_core.c:2206 (sleep mutex:256 Bucket) 0 0 174 0 1228 0 0 0 0 /usr/src/sys/vm/uma_core.c:2724 (sleep mutex:256 Bucket) 2 0 110 0 1200 0 0 0 0 /usr/src/sys/cam/cam_xpt.c:2160 (sleep mutex:CAM device lock) 0 0 53 0 1200 0 0 0 0 /usr/src/sys/cam/cam_xpt.c:2127 (sleep mutex:CAM bus lock) 0 0 52 0 1200 0 0 0 0 /usr/src/sys/cam/cam_xpt.c:2150 (sleep mutex:CAM bus lock) 0 0 49 0 1200 0 0 0 0 /usr/src/sys/cam/cam_xpt.c:1638 (sleep mutex:CAM bus lock) 0 0 48 0 1200 0 0 0 0 /usr/src/sys/cam/cam_xpt.c:2167 (sleep mutex:CAM bus lock) 3 0 892 0 1131 0 0 0 0 /usr/src/sys/kern/kern_timeout.c:364 (spin mutex:callout) 4 0 428 0 1091 0 0 0 2 /usr/src/sys/kern/sched_ule.c:1203 (spin mutex:sched lock 1) 0 1 126 25 1058 0 0 0 47 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 1) 0 0 325 7 1011 0 0 0 12 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 1) 1 2 348 4 959 0 0 0 4 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 22) 2 1 389 2 955 0 0 0 2 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 18) 2 0 338 1 950 0 0 0 3 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 20) 1 0 360 1 947 0 0 0 3 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 14) 3 0 362 3 925 0 0 0 6 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 19) 0 0 45 0 924 0 0 0 0 /usr/src/sys/cam/cam_xpt.c:5157 (sleep mutex:XPT topology lock) 0 2 149 4 921 0 0 0 4 /usr/src/sys/kern/subr_taskqueue.c:284 (spin mutex:mlx4_en_que) 1 0 442 1 920 0 0 0 5 /usr/src/sys/kern/subr_taskqueue.c:467 (spin mutex:mlx4_en_que) 2 0 288 0 904 0 0 0 0 /usr/src/sys/kern/kern_synch.c:309 (spin mutex:mlx4_en_que) 4 1 403 4 903 0 0 0 7 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 12) 2 0 366 0 894 0 0 0 2 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 23) 5 0 338 1 888 0 0 0 2 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 16) 3 0 338 0 885 0 0 0 0 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 17) 1 3 311 4 881 0 0 0 2 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 21) 2 3 343 8 864 0 0 0 5 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 13) 4 0 331 0 844 0 0 0 0 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 15) 0 0 121 0 715 0 0 0 1 /usr/src/sys/kern/subr_taskqueue.c:284 (spin mutex:mlx4_en_que) 2 0 335 1 713 0 0 0 5 /usr/src/sys/kern/subr_taskqueue.c:467 (spin mutex:mlx4_en_que) 68 0 4031 0 711 5 0 0 0 /usr/src/sys/netinet/tcp_timer.c:284 (rw:tcpinp) 4 0 177 0 699 0 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:5515 (rw:pmap pv list) 2 0 211 0 690 0 0 0 0 /usr/src/sys/kern/kern_synch.c:309 (spin mutex:mlx4_en_que) 4 0 172 0 678 0 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:3386 (rw:pmap pv list) 46 0 3880 0 597 6 0 0 0 /usr/src/sys/modules/mlxen/../../ofed/drivers/net/mlx4/en_tx.c:480 (sleep mutex:mlx4 comp) 0 0 166 0 572 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 1) 0 0 93 0 572 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 1) 413 0 4557 0 570 7 0 0 0 /usr/src/sys/kern/kern_proc.c:1450 (sleep mutex:process lock) 203 0 1822 0 570 3 0 0 0 /usr/src/sys/kern/kern_proc.c:957 (spin mutex:pstatl) 0 0 59 0 570 0 0 0 0 /usr/src/sys/kern/kern_proc.c:924 (sleep mutex:sigacts) 0 0 30 0 570 0 0 0 0 /usr/src/sys/kern/kern_proc.c:978 (sleep mutex:session) 1 0 63 0 564 0 0 0 0 /usr/src/sys/kern/kern_proc.c:301 (sleep mutex:process lock) 0 0 70 0 562 0 0 0 0 /usr/src/sys/kern/kern_proc.c:323 (sx:allproc) 0 0 110 0 547 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 1) 1 0 120 0 540 0 0 0 0 /usr/src/sys/kern/kern_proc.c:1030 (spin mutex:sleepq chain) 22 0 1864 0 531 3 0 0 0 /usr/src/sys/vm/vm_kern.c:232 (rw:kernel vm object) 19 0 138 0 531 0 0 0 0 /usr/src/sys/vm/vm_page.c:1897 (sleep mutex:vm page free queue) 2 0 179 0 531 0 0 0 0 /usr/src/sys/kern/subr_vmem.c:1141 (sleep mutex:kernel arena) 1 0 45 0 531 0 0 0 0 /usr/src/sys/vm/uma_core.c:1022 (sleep mutex:mbuf_jumbo_9k) 1 0 261 10 477 0 0 0 39 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 1) 0 0 199 0 477 0 0 0 0 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 1) 0 0 96 0 464 0 0 0 1 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 7) 0 0 97 0 463 0 0 0 0 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 8) 0 0 100 0 458 0 0 0 0 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 5) 0 0 99 0 458 0 0 0 1 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 3) 0 0 92 0 446 0 0 0 1 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 4) 0 0 97 0 445 0 0 0 1 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 6) 0 0 100 0 443 0 0 0 0 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 9) 0 1 116 1 440 0 0 0 1 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 11) 0 0 99 0 440 0 0 0 0 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 2) 0 1 44 14 430 0 0 0 13 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 0) 0 25 105 46 429 0 0 0 23 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 0) 1 2 53 9 427 0 0 0 11 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 0) 0 0 36 0 425 0 0 0 1 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 3) 0 0 35 1 423 0 0 0 3 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 7) 0 0 96 0 416 0 0 0 0 /usr/src/sys/kern/kern_switch.c:211 (spin mutex:sched lock 10) 0 0 34 1 410 0 0 0 3 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 6) 0 0 33 0 404 0 0 0 1 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 2) 0 0 32 1 402 0 0 0 4 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 5) 0 0 35 1 392 0 0 0 2 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 4) 0 0 32 0 390 0 0 0 1 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 9) 9 0 87 0 386 0 0 0 0 /usr/src/sys/vm/vm_object.c:529 (rw:vm object) 0 0 30 1 383 0 0 0 2 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 8) 0 0 32 0 382 0 0 0 1 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 11) 0 0 33 2 371 0 0 0 3 /usr/src/sys/kern/sched_ule.c:2189 (spin mutex:sched lock 10) 0 0 17 0 349 0 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:5694 (sleep mutex:pmap) 4 0 257 0 345 0 0 0 0 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 18) 1 1 155 1 345 0 0 0 1 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 18) 0 0 96 3 344 0 0 0 6 /usr/src/sys/kern/kern_clock.c:800 (spin mutex:sched lock 1) 0 0 21 0 327 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 22) 11 0 631 0 320 1 0 0 0 /usr/src/sys/modules/mlx4/../../ofed/drivers/net/mlx4/cmd.c:479 (sx:lnxmtx) 1 0 33 0 320 0 0 0 0 /usr/src/sys/modules/mlx4/../../ofed/drivers/net/mlx4/cmd.c:719 (sleep mutex:lnxspin) 0 0 42 0 320 0 0 0 0 /usr/src/sys/kern/kern_sema.c:94 (sleep mutex:lnxsema) 0 0 40 0 320 0 0 0 0 /usr/src/sys/modules/mlx4/../../ofed/drivers/net/mlx4/cmd.c:780 (sleep mutex:lnxspin) 0 0 33 0 320 0 0 0 0 /usr/src/sys/kern/kern_sema.c:79 (sleep mutex:lnxsema) 1 0 50 0 316 0 0 0 0 /usr/src/sys/vm/vm_glue.c:171 (sx:vm map (user)) 1 0 174 0 309 0 0 0 2 /usr/src/sys/kern/subr_taskqueue.c:467 (spin mutex:mlx4_en_que) 0 0 55 0 309 0 0 0 0 /usr/src/sys/kern/subr_taskqueue.c:284 (spin mutex:mlx4_en_que) 1 0 84 0 303 0 0 0 0 /usr/src/sys/kern/kern_synch.c:309 (spin mutex:mlx4_en_que) 2 1 64 1 297 0 0 0 2 /usr/src/sys/vm/vm_page.c:1651 (sleep mutex:vm page free queue) 4 0 344 0 295 1 0 0 0 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 0) 98 0 2397 0 286 8 0 0 0 /usr/src/sys/netinet/tcp_timer.c:587 (rw:tcpinp) 12 0 307 0 286 1 0 0 0 /usr/src/sys/netinet/tcp_timer.c:584 (rw:tcp) 0 0 18 0 282 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 23) 20 0 845 0 281 3 0 0 0 /usr/src/sys/vm/vm_map.c:3985 (sx:vm map (user)) 1 0 78 0 269 0 0 0 0 /usr/src/sys/vm/vm_fault.c:1010 (rw:vm object) 1 0 43 0 269 0 0 0 0 /usr/src/sys/vm/vm_fault.c:1011 (sleep mutex:vm page) 0 0 18 0 269 0 0 0 0 /usr/src/sys/vm/vm_page.c:2796 (sleep mutex:vm active pagequeue) 0 0 17 0 269 0 0 0 0 /usr/src/sys/vm/vm_page.c:2840 (sleep mutex:vm active pagequeue) 1 0 27 0 247 0 0 0 0 /usr/src/sys/vm/vm_page.c:3037 (sleep mutex:vm page free queue) 0 0 22 0 246 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 13) 0 0 21 0 228 0 0 0 0 /usr/src/sys/vm/vm_page.c:1437 (sleep mutex:vm page) 0 0 8 0 200 0 0 0 0 /usr/src/sys/cam/cam_xpt.c:2112 (sleep mutex:CAM bus lock) 0 0 8 0 200 0 0 0 0 /usr/src/sys/cam/cam_xpt.c:1604 (sleep mutex:CAM bus lock) 0 0 22 0 197 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 7) 9 0 203 0 185 1 0 0 0 /usr/src/sys/vm/vm_fault.c:373 (rw:vm object) 1 0 116 6 178 0 0 0 13 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 9) 3 0 175 7 173 1 0 0 17 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 2) 1 0 113 6 173 0 0 0 16 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 11) 1 0 40 0 173 0 0 0 0 /usr/src/sys/vm/vm_fault.c:503 (sleep mutex:vm page) 0 0 18 0 173 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 8) 0 0 14 0 173 0 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:4460 (rw:pmap pv list) 0 0 13 0 173 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 18) 0 0 18 0 168 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 2) 4 0 170 0 162 1 0 0 0 /usr/src/sys/kern/subr_taskqueue.c:467 (sleep mutex:mlx4_en) 1 5 39 5 162 0 0 0 2 /usr/src/sys/kern/subr_taskqueue.c:284 (sleep mutex:mlx4_en) 0 0 19 0 162 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 3) 0 0 14 0 161 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 14) 0 0 17 0 158 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 10) 0 0 10 0 156 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:509 (sx:zio->io_lock) 1 0 49 0 152 0 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:5984 (sleep mutex:pmap) 0 0 17 0 151 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 6) 0 0 9 0 151 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 15) 1 1 92 8 150 0 0 0 19 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 10) 9 0 142 0 146 0 0 0 0 /usr/src/sys/vm/vm_pageout.c:336 (sleep mutex:vm page) 8 0 97 0 146 0 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:5879 (rw:pmap pv list) 1 1 81 35 141 0 0 0 49 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 11) 0 0 15 0 137 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 4) 0 0 10 0 136 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 20) 1 0 96 4 134 0 0 0 12 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 7) 0 0 17 0 134 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 11) 0 0 10 0 128 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 19) 0 0 14 0 127 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 5) 2 3 97 38 125 0 0 0 41 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 2) 1 0 84 1 125 0 0 0 4 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 3) 2 3 84 5 123 0 0 0 2 /usr/src/sys/kern/kern_mutex.c:190 (sleep mutex:mlx4_en) 1 0 14 0 121 0 0 0 0 /usr/src/sys/cam/scsi/scsi_enc_ses.c:1072 (sleep mutex:CAM device lock) 0 0 15 0 120 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 9) 1 1 79 3 119 0 0 0 3 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 21) 2 1 95 22 118 0 0 0 30 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 4) 2 0 115 0 118 0 0 0 0 /usr/src/sys/net/bpf.c:2206 (rw:bpf interface lock) 1 0 86 0 118 0 0 0 0 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 21) 0 0 23 0 118 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/rrwlock.c:242 (sx:rrl->rr_lock) 2 0 84 0 117 0 0 0 0 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 23) 1 0 87 0 117 0 0 0 0 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 23) 3 0 131 5 116 1 0 0 12 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 4) 2 0 93 0 116 0 0 0 0 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 22) 1 2 62 39 116 0 0 0 40 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 10) 1 0 6 0 116 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/rrwlock.c:165 (sx:rrl->rr_lock) 2 3 92 31 115 0 0 0 35 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 6) 2 0 77 2 115 0 0 0 9 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 5) 1 1 91 1 115 0 0 0 1 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 22) 0 0 9 0 114 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 12) 3 0 86 0 111 0 0 0 0 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 16) 1 0 79 0 111 0 0 0 0 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 16) 1 0 75 3 109 0 0 0 9 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 8) 31 2 441 6 108 4 0 0 3 /usr/src/sys/dev/usb/usb_transfer.c:2265 (sleep mutex:USB device mutex) 1 0 12 0 108 0 0 0 0 /usr/src/sys/cam/cam_xpt.c:3445 (sleep mutex:CAM bus lock) 0 0 35 0 107 0 0 0 0 /usr/src/sys/vm/vm_object.c:763 (sleep mutex:vm page) 1 3 63 31 106 0 0 0 35 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 9) 4 0 88 0 104 0 0 0 0 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 13) 2 0 86 0 104 0 0 0 0 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 13) 3 0 84 0 103 0 0 0 0 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 17) 2 2 88 34 103 0 0 0 42 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 8) 1 1 78 3 103 0 0 0 2 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 17) 1 1 76 2 102 0 0 0 2 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 20) 1 0 77 0 101 0 0 0 0 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 20) 2 0 80 0 100 0 0 0 0 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 15) 1 1 80 1 99 0 0 0 1 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 15) 0 0 20 0 99 0 0 0 0 /usr/src/sys/vm/uma_core.c:2206 (sleep mutex:mlx4_cmd) 3 0 115 2 98 1 0 0 5 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 8) 1 0 76 0 98 0 0 0 0 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 19) 1 0 71 0 98 0 0 0 0 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 19) 1 1 55 3 97 0 0 0 6 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 4) 0 0 15 0 97 0 0 0 0 /usr/src/sys/vm/uma_core.c:2724 (sleep mutex:mlx4_cmd) 4 0 52 0 96 0 0 0 0 /usr/src/sys/vm/vm_fault.c:409 (rw:vm object) 3 0 102 4 96 1 0 0 9 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 6) 2 0 20 0 96 0 0 0 0 /usr/src/sys/vm/vm_object.c:2244 (sleep mutex:vm page) 2 0 101 0 95 1 0 0 1 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 20) 1 1 92 2 95 0 0 0 3 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 12) 1 0 73 0 94 0 0 0 0 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 12) 3 0 90 1 92 0 0 0 2 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 21) 2 0 73 0 91 0 0 0 0 /usr/src/sys/kern/kern_intr.c:1365 (spin mutex:sched lock 14) 1 1 79 1 90 0 0 0 1 /usr/src/sys/kern/kern_intr.c:912 (spin mutex:sched lock 14) 1 1 63 19 90 0 0 0 35 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 1) 1 0 55 5 90 0 0 0 13 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 6) 1 0 52 0 90 0 0 0 0 /usr/src/sys/netinet/if_ether.c:876 (rw:if_addr_lock) 1 0 49 0 90 0 0 0 0 /usr/src/sys/netinet/if_ether.c:943 (rw:if_afdata) 0 0 19 0 90 0 0 0 0 /usr/src/sys/netinet/if_ether.c:1023 (rw:if_afdata) 1 2 58 15 89 0 0 0 22 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 7) 3 0 86 0 86 1 0 0 0 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 22) 0 0 14 0 84 0 0 0 1 /usr/src/sys/kern/kern_sig.c:1027 (sleep mutex:process lock) 5 0 101 0 82 1 0 0 1 /usr/src/sys/kern/kern_mutex.c:190 (sleep mutex:process lock) 0 0 6 0 82 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 21) 8 0 32 0 81 0 0 0 0 /usr/src/sys/vm/vm_map.c:2931 (rw:vm object) 939 0 48463 0 80 605 0 0 0 /usr/src/sys/modules/mlxen/../../ofed/drivers/net/mlx4/en_netdev.c:1150 (sx:lnxmtx) 197 0 14283 0 80 178 0 0 0 /usr/src/sys/modules/mlx4/../../ofed/drivers/net/mlx4/main.c:2655 (sx:lnxmtx) 10 0 320 0 80 4 0 0 0 /usr/src/sys/modules/mlxen/../../ofed/drivers/net/mlx4/en_port.c:182 (sleep mutex:lnxspin) 2 0 78 0 80 0 0 0 1 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 12) 2 0 71 0 80 0 0 0 0 /usr/src/sys/modules/mlxen/../../ofed/drivers/net/mlx4/en_port.c:530 (sleep mutex:lnxspin) 2 0 66 0 80 0 0 0 0 /usr/src/sys/modules/mlxen/../../ofed/drivers/net/mlx4/en_netdev.c:1175 (sx:lnxmtx) 1 0 54 0 80 0 0 0 0 /usr/src/sys/kern/kern_resource.c:616 (spin mutex:pstatl) 2 0 92 0 79 1 0 0 1 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 15) 1 1 50 25 79 0 0 0 28 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 3) 1 0 27 0 78 0 0 0 0 /usr/src/sys/kern/kern_mutex.c:190 (sleep mutex:ehci1) 6 2 172 14 77 2 0 0 6 /usr/src/sys/vm/vm_fault.c:793 (rw:vm object) 2 0 23 0 77 0 0 0 0 /usr/src/sys/vm/vm_fault.c:139 (sleep mutex:vm page) 0 2 5 5 77 0 0 0 3 /usr/src/sys/vm/vm_page.c:3185 (sleep mutex:vm inactive pagequeue) 0 0 5 0 77 0 0 0 0 /usr/src/sys/vm/vm_fault.c:914 (rw:vm object) 4 0 85 4 76 1 0 0 9 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 3) 4 0 76 0 76 1 0 0 2 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 19) 0 0 6 0 76 0 0 0 0 /usr/src/sys/vm/vm_page.c:2796 (sleep mutex:vm inactive pagequeue) 4 0 84 0 75 1 0 0 1 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 13) 2 0 67 1 75 0 0 0 2 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 16) 3 0 78 0 73 1 0 0 0 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 23) 2 0 86 0 73 1 0 0 0 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 17) 1 0 20 0 73 0 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:4482 (rw:pmap pv list) 6 0 68 0 72 0 0 0 0 /usr/src/sys/dev/usb/usb_transfer.c:2411 (sleep mutex:ehci1) 1 0 19 0 72 0 0 0 0 /usr/src/sys/dev/usb/usb_transfer.c:2266 (sleep mutex:ehci1) 0 0 7 0 72 0 0 0 1 /usr/src/sys/dev/usb/usb_transfer.c:2013 (sleep mutex:ehci1) 0 0 6 0 72 0 0 0 0 /usr/src/sys/dev/usb/usb_hub.c:2371 (sleep mutex:ehci1) 3 0 71 0 71 1 0 0 2 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 14) 0 0 17 0 71 0 0 0 0 /usr/src/sys/kern/sched_ule.c:1933 (spin mutex:sched lock 0) 0 0 6 0 70 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 13) 3 0 74 3 68 1 0 0 8 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 5) 1 2 49 18 68 0 0 0 21 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 5) 0 2 34 20 67 0 0 0 21 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 8) 1556 0 8389 0 66 127 0 0 0 /usr/src/sys/dev/usb/usb_device.c:2798 (sx:USB suspend and resume SX lock) 3 0 18 0 66 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_acl.c:2100 (sx:zp->z_acl_lock) 2 1 60 19 65 0 0 0 26 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 10) 0 0 13 0 65 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:357 (spin mutex:sched lock 0) 1 2 50 18 64 0 0 0 17 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 5) 1 0 35 1 64 0 0 0 4 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 2) 4 0 77 3 62 1 0 0 9 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 7) 19 0 207 0 60 3 0 0 0 /usr/src/sys/kern/vfs_subr.c:2523 (lockmgr:zfs) 3 0 69 0 60 1 0 0 0 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 18) 0 0 5 0 60 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 17) 1 2 36 17 59 0 0 0 19 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 6) 0 0 6 0 58 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 16) 0 0 7 0 57 0 0 0 0 /usr/src/sys/vm/vm_glue.c:756 (sleep mutex:process lock) 2 0 21 0 56 0 0 0 0 /usr/src/sys/kern/vfs_cache.c:585 (rw:Name Cache) 1 1 45 12 55 0 0 0 17 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 3) 12 0 178 0 54 3 0 0 0 /usr/src/sys/dev/usb/usb_request.c:544 (sleep mutex:USB device mutex) 7 8 205 51 54 3 0 0 12 /usr/src/sys/kern/kern_condvar.c:150 (sleep mutex:Giant) 1 3 46 23 54 0 0 0 22 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 9) 1 0 45 0 54 0 0 0 0 /usr/src/sys/kern/kern_mutex.c:190 (sleep mutex:USB device mutex) 0 0 5 0 54 0 0 0 0 /usr/src/sys/dev/usb/usb_transfer.c:3102 (sleep mutex:USB device mutex) 2 0 67 0 52 1 0 0 1 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 10) 0 0 2 0 51 0 0 0 0 /usr/src/sys/vm/swap_pager.c:307 (sleep mutex:ui_vmsize) 0 0 2 0 51 0 0 0 0 /usr/src/sys/vm/swap_pager.c:301 (sleep mutex:swapdev) 2 0 53 0 50 1 0 0 0 /usr/src/sys/kern/kern_mutex.c:190 (sleep mutex:msgbuf lock) 5 0 179 0 48 3 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/txg.c:367 (sx:tx->tx_cpu[c].tc_open_lock) 0 0 4 0 48 0 0 0 0 /usr/src/sys/kern/sys_generic.c:1777 (sleep mutex:select mtxpool) 0 0 3 0 48 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/txg.c:388 (sx:tx->tx_cpu[c].tc_lock) 0 0 10 0 47 0 0 0 0 /usr/src/sys/kern/kern_sig.c:665 (sleep mutex:process lock) 0 0 4 0 47 0 0 0 0 /usr/src/sys/kern/kern_sig.c:667 (sleep mutex:sigacts) 0 0 8 0 46 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:2926 (sleep mutex:vnode interlock) 0 0 6 0 46 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:2627 (sleep mutex:vnode interlock) 0 0 3 0 46 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:2541 (sleep mutex:vnode interlock) 0 0 3 0 46 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c:4288 (sx:zfsvfs->z_teardown_inactive_lock) 1 1 34 10 45 0 0 0 13 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 7) 2 0 53 0 44 1 0 0 1 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 16) 0 3 15 53 43 0 1 0 34 /usr/src/sys/vm/uma_core.c:2208 (sleep mutex:mbuf) 0 0 14 0 43 0 0 0 2 /usr/src/sys/kern/sched_ule.c:2645 (spin mutex:sched lock 1) 0 0 5 1 43 0 0 0 3 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 9) 20 318 167 318 42 3 7 0 2 /usr/src/sys/dev/random/fortuna.c:356 (sleep mutex:reseed mutex) 5 0 55 0 42 1 0 0 0 /usr/src/sys/dev/random/fortuna.c:412 (sleep mutex:reseed mutex) 1 1 26 4 42 0 0 0 7 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 4) 1 0 19 0 42 0 0 0 0 /usr/src/sys/kern/kern_mutex.c:190 (sleep mutex:ehci0) 0 0 7 0 42 0 0 0 0 /usr/src/sys/kern/sys_generic.c:1708 (sleep mutex:select mtxpool) 0 0 2 0 42 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:2668 (sleep mutex:vnode interlock) 3 0 46 2 41 1 0 0 8 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 9) 2 0 9 0 41 0 0 0 0 /usr/src/sys/vm/vm_fault.c:1186 (rw:vm object) 2 0 46 0 40 1 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 18) 1 0 17 0 40 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/multilist.c:257 (sx:mls->mls_lock) 3 3 33 18 39 0 0 0 15 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 23) 3 1 7 1 39 0 0 0 2 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 9) 1 3 26 36 39 0 0 0 26 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 18) 38 0 205 0 38 5 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:5018 (sleep mutex:pmap) 38 0 201 0 38 5 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:5019 (sleep mutex:pmap) 2 0 45 0 38 1 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 17) 0 0 9 0 38 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:443 (spin mutex:sched lock 0) 0 0 5 1 38 0 0 0 2 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 10) 8 0 70 0 37 1 0 0 1 /usr/src/sys/kern/subr_turnstile.c:541 (spin mutex:turnstile chain) 8 0 62 0 36 1 0 0 0 /usr/src/sys/dev/usb/usb_transfer.c:1891 (sleep mutex:ehci1) 7 0 91 0 36 2 0 0 0 /usr/src/sys/dev/usb/controller/ehci.c:1465 (sleep mutex:ehci0) 6 0 134 0 36 3 0 0 0 /usr/src/sys/dev/usb/controller/ehci.c:1465 (sleep mutex:ehci1) 6 0 80 0 36 2 0 0 0 /usr/src/sys/dev/usb/usb_transfer.c:1837 (sleep mutex:ehci1) 2 0 43 0 36 1 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 19) 2 0 34 0 36 0 0 0 0 /usr/src/sys/dev/usb/usb_transfer.c:2411 (sleep mutex:ehci0) 1 0 9 0 36 0 0 0 0 /usr/src/sys/vm/vm_map.c:3171 (rw:vm object) 0 0 11 0 36 0 0 0 0 /usr/src/sys/dev/usb/usb_transfer.c:2266 (sleep mutex:ehci0) 0 0 6 0 36 0 0 0 0 /usr/src/sys/dev/usb/usb_transfer.c:2995 (sleep mutex:ehci1) 0 0 5 0 36 0 0 0 0 /usr/src/sys/x86/x86/io_apic.c:276 (spin mutex:icu) 0 0 4 0 36 0 0 0 0 /usr/src/sys/x86/x86/io_apic.c:259 (spin mutex:icu) 0 0 4 0 36 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 12) 0 0 4 0 36 0 0 0 0 /usr/src/sys/dev/usb/usb_hub.c:2371 (sleep mutex:ehci0) 0 0 3 0 36 0 0 0 0 /usr/src/sys/dev/usb/usb_transfer.c:2013 (sleep mutex:ehci0) 0 0 2 0 36 0 0 0 0 /usr/src/sys/dev/usb/usb_transfer.c:2402 (sleep mutex:ehci1) 3 2 31 16 35 0 0 0 16 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 19) 2 0 41 0 35 1 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 20) 1 1 24 14 35 0 0 0 15 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 22) 1 0 17 0 35 0 0 0 0 /usr/src/sys/vm/vm_object.c:1912 (sleep mutex:vm page) 0 1 5 2 35 0 0 0 3 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 10) 0 0 4 0 35 0 0 0 1 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 18) 0 0 3 0 35 0 0 0 0 /usr/src/sys/vm/vm_object.c:435 (rw:vm object) 0 0 3 0 35 0 0 0 0 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 18) 7 0 90 0 34 2 0 0 0 /usr/src/sys/kern/sys_pipe.c:1209 (sleep mutex:pipe mutex) 1 0 19 0 34 0 0 0 0 /usr/src/sys/kern/sys_pipe.c:1053 (sleep mutex:pipe mutex) 1 0 16 0 34 0 0 0 0 /usr/src/sys/kern/sched_ule.c:905 (spin mutex:sched lock 1) 1566 0 15204 0 33 460 0 0 0 /usr/src/sys/dev/usb/usb_device.c:2751 (sx:USB config SX lock) 4 0 27 0 33 0 0 0 0 /usr/src/sys/dev/usb/usb_device.c:2752 (sx:USB suspend and resume SX lock) 3 0 39 0 33 1 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 23) 2 1 32 9 33 0 0 0 10 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 17) 0 0 7 0 33 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 14) 2 0 39 0 32 1 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 22) 2 0 35 0 32 1 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 13) 2 0 33 0 32 1 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 21) 0 0 3 0 32 0 0 0 0 /usr/src/sys/kern/kern_umtx.c:3747 (sleep mutex:umtxshm) 0 2 5 3 31 0 0 0 2 /usr/src/sys/vm/vm_fault.c:1193 (rw:vm object) 0 0 4 0 31 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 15) 0 0 3 0 31 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 23) 0 0 3 0 31 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 21) 8 0 121 0 30 4 0 0 0 /usr/src/sys/kern/tty_ttydisc.c:480 (sleep mutex:ttymtx) 3 0 22 0 30 0 0 0 0 /usr/src/sys/kern/sys_pipe.c:708 (sleep mutex:pipe mutex) 3 0 12 0 30 0 0 0 0 /usr/src/sys/kern/kern_sysctl.c:164 (sleep mutex:Giant) 2 4 23 34 30 0 1 0 18 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 21) 2 0 35 0 30 1 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 15) 2 0 29 0 30 0 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 14) 1 2 6 2 30 0 0 0 1 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 6) 1 0 15 0 30 0 0 0 0 /usr/src/sys/kern/sys_pipe.c:671 (sleep mutex:pipe mutex) 1 0 8 0 30 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 15) 0 2 7 24 30 0 0 0 25 /usr/src/sys/vm/uma_core.c:2726 (sleep mutex:mbuf) 0 2 4 2 30 0 0 0 1 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 8) 0 0 14 0 30 0 0 0 0 /usr/src/sys/vm/vm_object.c:1605 (sleep mutex:vm page) 0 0 8 0 30 0 0 0 0 /usr/src/sys/kern/kern_kthread.c:214 (sleep mutex:process lock) 0 0 4 0 30 0 0 0 0 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 8) 1 0 10 0 29 0 0 0 0 /usr/src/sys/kern/tty_outq.c:244 (sleep mutex:ttymtx) 0 1 3 2 29 0 0 0 2 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 12) 0 0 7 0 29 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 12) 3 2 38 15 28 1 0 0 12 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 12) 2 1 27 5 28 0 0 0 7 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 11) 1 3 20 10 28 0 0 0 9 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 20) 1 0 31 0 28 1 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 12) 1 0 7 0 28 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sa.c:1891 (sx:hdl->sa_lock) 1 0 5 1 28 0 0 0 2 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 4) 0 3 6 23 28 0 0 0 20 /usr/src/sys/vm/uma_core.c:2726 (sleep mutex:mbuf_jumbo_page) 0 1 4 1 28 0 0 0 1 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 12) 1 2 12 2 27 0 0 0 1 /usr/src/sys/kern/kern_mutex.c:190 (sleep mutex:pipe mutex) 0 1 2 1 27 0 0 0 2 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 16) 0 0 4 0 27 0 0 0 0 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 4) 0 0 3 1 27 0 0 0 3 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 3) 23 0 70 0 26 2 0 0 0 /usr/src/sys/kern/vfs_lookup.c:530 (lockmgr:zfs) 0 3 3 7 26 0 0 0 5 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 11) 0 1 8 2 26 0 0 0 2 /usr/src/sys/kern/tty_pts.c:392 (sleep mutex:ttymtx) 0 0 15 0 26 0 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 8) 0 0 6 0 26 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 4) 0 0 4 0 26 0 0 0 1 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 11) 0 0 3 0 26 0 0 0 0 /usr/src/sys/kern/vfs_lookup.c:235 (sx:filedesc structure) 0 0 3 0 26 0 0 0 0 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 22) 0 0 3 0 26 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:3198 (sx:zio->io_lock) 0 0 2 0 26 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:3486 (sx:zio->io_lock) 0 0 2 0 26 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/ddt.c:637 (sx:ddt->ddt_lock) 1 0 14 0 25 0 0 0 0 /usr/src/sys/kern/tty.c:171 (sleep mutex:ttymtx) 0 0 4 0 25 0 0 0 0 /usr/src/sys/kern/kern_conf.c:72 (sleep mutex:cdev) 0 0 3 0 25 0 0 0 1 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 20) 0 0 3 0 25 0 0 0 0 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 2) 0 0 3 0 25 0 0 0 0 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 6) 0 0 3 0 25 0 0 0 0 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 2) 0 0 2 0 25 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 14) 4 2 31 17 24 1 0 0 15 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 18) 2 2 24 14 24 1 0 0 12 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 13) 0 0 5 0 24 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 6) 0 0 4 0 24 0 0 0 0 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 3) 0 0 3 0 24 0 0 0 1 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 14) 0 0 3 0 24 0 0 0 1 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 20) 0 0 3 0 24 0 0 0 1 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 16) 0 0 1 0 24 0 0 0 0 /usr/src/sys/vm/vm_object.c:1266 (rw:vm object) 0 0 1 0 24 0 0 0 0 /usr/src/sys/vm/vm_object.c:1233 (rw:vm object) 0 0 1 0 24 0 0 0 0 /usr/src/sys/vm/vm_map.c:4084 (rw:vm object) 3 2 22 8 23 0 0 0 7 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 15) 3 0 48 0 23 2 0 0 0 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 18) 1 0 4 0 23 0 0 0 1 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 22) 0 0 5 0 23 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 3) 0 0 2 0 23 0 0 0 1 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 5) 0 0 2 0 23 0 0 0 0 /usr/src/sys/vm/swap_pager.c:211 (sleep mutex:swapdev) 0 0 2 0 23 0 0 0 0 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 5) 0 0 1 0 23 0 0 0 0 /usr/src/sys/vm/swap_pager.c:227 (sleep mutex:ui_vmsize) 1 0 10 0 22 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:3498 (sx:zio->io_lock) 0 0 14 0 22 0 0 0 1 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 4) 0 0 13 0 22 0 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 6) 0 0 7 0 22 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 16) 0 0 5 0 22 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 0) 0 0 5 0 22 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 17) 0 0 3 0 22 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:1582 (sx:zio->io_lock) 3 0 44 0 21 2 0 0 0 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 12) 0 3 6 23 21 0 1 0 19 /usr/src/sys/vm/uma_core.c:2208 (sleep mutex:mbuf_jumbo_page) 0 0 5 0 21 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:645 (spin mutex:sched lock 1) 0 0 5 0 21 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 22) 0 0 5 0 21 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 20) 0 0 3 0 21 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 22) 0 0 3 0 21 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 11) 0 0 2 0 21 0 0 0 0 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 14) 0 0 1 0 21 0 0 0 0 /usr/src/sys/vm/vm_mmap.c:1482 (sleep mutex:process lock) 26 0 87 0 20 4 0 0 0 /usr/src/sys/vm/vm_object.c:1693 (rw:vm object) 20 0 178 0 20 8 0 0 0 /usr/src/sys/vm/vm_pageout.c:1199 (sleep mutex:vm active pagequeue) 9 0 23 0 20 1 0 0 0 /usr/src/sys/netinet6/mld6.c:1746 (sleep mutex:mld_mtx) 3 0 22 0 20 1 0 0 0 /usr/src/sys/kern/kern_mutex.c:190 (sleep mutex:Giant) 2 0 13 0 20 0 0 0 0 /usr/src/sys/netinet/igmp.c:2186 (sleep mutex:igmp_mtx) 0 1 7 2 20 0 0 0 4 /usr/src/sys/kern/kern_proc.c:1030 (spin mutex:sched lock 0) 0 0 8 0 20 0 0 0 0 /usr/src/sys/vm/vm_pageout.c:951 (sleep mutex:vm inactive pagequeue) 0 0 8 0 20 0 0 0 0 /usr/src/sys/vm/vm_pageout.c:1552 (sleep mutex:vm page free queue) 0 0 7 0 20 0 0 0 0 /usr/src/sys/netinet6/frag6.c:741 (sleep mutex:ip6qlock) 0 0 5 0 20 0 0 0 0 /usr/src/sys/netinet/tcp_timewait.c:691 (rw:tcptw) 0 0 5 0 20 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 7) 9 0 29 0 19 1 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:3776 (sleep mutex:pmap) 2 0 6 0 19 0 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:484 (sleep mutex:invlgn) 1 0 11 0 19 0 0 0 0 /usr/src/sys/kern/uipc_socket.c:3026 (sleep mutex:so_snd) 0 4 10 8 19 0 0 0 8 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 12) 0 0 11 0 19 0 0 0 1 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 10) 0 0 5 0 19 0 0 0 0 /usr/src/sys/kern/uipc_socket.c:3027 (sleep mutex:so_rcv) 0 0 5 0 19 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 8) 0 0 3 0 19 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 20) 0 0 3 0 19 0 0 0 0 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 7) 0 0 1 0 19 0 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:452 (sleep mutex:invlgn) 25 0 79 0 18 4 0 0 0 /usr/src/sys/dev/usb/usb_transfer.c:1837 (sleep mutex:ehci0) 7 1 55 5 18 3 0 0 8 /usr/src/sys/kern/subr_turnstile.c:598 (spin mutex:turnstile lock) 6 0 39 0 18 2 0 0 0 /usr/src/sys/dev/usb/usb_transfer.c:1891 (sleep mutex:ehci0) 1 0 6 0 18 0 0 0 0 /usr/src/sys/vm/uma_core.c:2206 (sleep mutex:tcpreass) 1 0 6 0 18 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:836 (spin mutex:td_contested) 0 0 7 0 18 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:5263 (sleep mutex:vnode interlock) 0 0 6 0 18 0 0 0 0 /usr/src/sys/vm/uma_core.c:2724 (sleep mutex:tcpreass) 0 0 5 0 18 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:5302 (sleep mutex:vnode_free_list) 0 0 5 0 18 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:896 (spin mutex:td_contested) 0 0 5 0 18 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:712 (spin mutex:td_contested) 0 0 3 0 18 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 2) 0 0 3 0 18 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 23) 0 0 2 0 18 0 0 0 1 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 21) 0 0 2 0 18 0 0 0 1 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 21) 0 0 2 0 18 0 0 0 0 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 7) 0 0 2 0 18 0 0 0 0 /usr/src/sys/dev/usb/usb_transfer.c:2995 (sleep mutex:ehci0) 0 0 1 0 18 0 0 0 0 /usr/src/sys/dev/usb/usb_transfer.c:2402 (sleep mutex:ehci0) 4 0 29 0 17 1 0 0 0 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 11) 3 0 35 0 17 2 0 0 0 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 16) 1 0 9 0 17 0 0 0 0 /usr/src/sys/kern/sys_pipe.c:1435 (sleep mutex:pipe mutex) 0 0 11 0 17 0 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 3) 0 0 10 0 17 0 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 5) 0 0 5 0 17 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 10) 0 0 5 0 17 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 19) 0 0 4 0 17 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 21) 0 0 3 0 17 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 19) 3 0 15 0 16 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:2401 (sx:db->db_mtx) 2 3 25 5 16 1 0 0 5 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 16) 0 0 9 0 16 0 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 7) 0 0 7 0 16 0 0 0 0 /usr/src/sys/kern/kern_mutex.c:190 (sleep mutex:sellck) 0 0 7 0 16 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c:523 (sx:scl->scl_lock) 0 0 5 0 16 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c:496 (sx:scl->scl_lock) 0 0 4 0 16 0 0 0 0 /usr/src/sys/vm/uma_core.c:1022 (sleep mutex:mbuf) 0 0 3 0 16 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 18) 0 0 1 0 16 0 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:3121 (sleep mutex:pmap pv chunk list) 0 0 1 0 16 0 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:3046 (sleep mutex:pmap pv chunk list) 2 0 23 0 15 1 0 0 0 /usr/src/sys/kern/sys_generic.c:1869 (sleep mutex:sellck) 1 3 10 15 15 0 1 0 9 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 16) 1 0 15 0 15 1 0 0 0 /usr/src/sys/vm/vm_map.c:1505 (sx:vm map (user)) 0 0 8 0 15 0 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 2) 0 0 4 0 15 0 0 0 0 /usr/src/sys/vm/uma_core.c:2206 (sleep mutex:32 Bucket) 0 0 3 0 15 0 0 0 0 /usr/src/sys/vm/uma_core.c:2724 (sleep mutex:32 Bucket) 0 0 3 0 15 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 5) 5 0 19 0 14 1 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:4067 (sleep mutex:pmap) 1 0 9 0 14 0 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 9) 0 4 8 10 14 0 0 0 5 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 14) 0 0 5 0 14 0 0 0 0 /usr/src/sys/kern/subr_vmem.c:1331 (sleep mutex:kmem arena) 0 0 3 0 14 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 13) 0 0 2 0 14 0 0 0 0 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 1) 0 0 2 0 14 0 0 0 0 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 19) 0 0 2 0 14 0 0 0 0 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 1) 0 0 1 0 14 0 0 0 0 /usr/src/sys/vm/vm_object.c:659 (rw:vm object) 0 0 1 0 14 0 0 0 0 /usr/src/sys/vm/vm_map.c:1262 (rw:vm object) 0 0 1 0 14 0 0 0 0 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 17) 2 1 20 3 13 1 0 0 5 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 14) 0 0 2 0 13 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 17) 0 0 2 0 13 0 0 0 0 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 15) 0 0 1 0 13 0 0 0 0 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 19) 0 0 1 0 13 0 0 0 0 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 15) 29 0 79 0 12 6 0 0 0 /usr/src/sys/vm/vm_map.c:1875 (rw:vm object) 22 0 58 0 12 4 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:4636 (sleep mutex:pmap) 15 19 68 52 12 5 4 0 5 /usr/src/sys/dev/usb/usb_device.c:2804 (sleep mutex:Giant) 6 0 40 0 12 3 0 0 0 /usr/src/sys/kern/sys_generic.c:1824 (sleep mutex:select mtxpool) 2 2 15 12 12 1 1 0 9 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 22) 1 3 7 6 12 0 0 0 2 /usr/src/sys/dev/usb/usb_device.c:2758 (sleep mutex:Giant) 1 0 5 0 12 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:136 (sx:h->hash_mutexes[i]) 1 0 3 0 12 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:720 (sx:db->db_mtx) 0 0 7 0 12 0 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 11) 0 0 1 0 12 0 0 0 0 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 17) 0 0 1 0 12 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c:5059 (sx:buf->b_evict_lock) 0 0 0 0 12 0 0 0 0 /usr/src/sys/vm/vm_map.c:4112 (rw:vm object) 0 0 0 0 12 0 0 0 0 /usr/src/sys/vm/vm_fault.c:208 (sleep mutex:vm page) 0 0 0 0 12 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/txg.c:808 (sx:tl->tl_lock) 4 0 25 0 11 2 0 0 0 /usr/src/sys/vm/vm_mmap.c:550 (sx:vm map (user)) 0 0 5 1 11 0 0 0 2 /usr/src/sys/kern/sched_ule.c:902 (spin mutex:sched lock 2) 0 0 2 0 11 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 18) 0 0 1 0 11 0 0 0 1 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 23) 0 0 1 0 11 0 0 0 1 /usr/src/sys/kern/kern_proc.c:855 (spin mutex:sched lock 13) 0 0 1 0 11 0 0 0 0 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 23) 0 0 1 0 11 0 0 0 0 /usr/src/sys/kern/kern_resource.c:1042 (spin mutex:sched lock 13) 766 0 4818 0 10 481 0 0 0 /usr/src/sys/kern/kern_proc.c:1439 (sx:proctree) 766 0 4815 0 10 481 0 0 0 /usr/src/sys/kern/kern_proc.c:1440 (sx:allproc) 27 0 92 0 10 9 0 0 0 /usr/src/sys/vm/vm_object.c:587 (rw:vm object) 5 0 33 0 10 3 0 0 0 /usr/src/sys/kern/kern_sx.c:195 (sx:l2arc_feed_thr_lock) 5 0 24 0 10 2 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c:3752 (sx:arc_reclaim_lock) 4 0 25 0 10 2 0 0 0 /usr/src/sys/kern/kern_mutex.c:190 (sleep mutex:Syncer mtx) 4 0 18 0 10 1 0 0 0 /usr/src/sys/kern/kern_sx.c:195 (sx:arc_reclaim_lock) 3 1 14 4 10 1 0 0 5 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 20) 3 0 26 0 10 2 0 0 0 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 14) 3 0 22 0 10 2 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c:3819 (sx:arc_user_evicts_lock) 3 0 20 0 10 2 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/trim_map.c:564 (sx:spa->spa_trim_lock) 3 0 16 0 10 1 0 0 0 /usr/src/sys/kern/subr_eventhandler.c:251 (sleep mutex:eventhandler) 2 0 16 0 10 1 0 0 0 /usr/src/sys/kern/kern_mutex.c:190 (sleep mutex:MPT2SAS lock) 2 0 14 0 10 1 0 0 0 /usr/src/sys/kern/vfs_bio.c:3038 (sleep mutex:buffer daemon lock) 2 0 12 0 10 1 0 0 0 /usr/src/sys/kern/vfs_subr.c:1066 (sleep mutex:vnode_free_list) 1 0 5 0 10 0 0 0 0 /usr/src/sys/kern/kern_mutex.c:190 (sleep mutex:buffer daemon lock) 1 0 4 0 10 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/trim_map.c:500 (sx:tm->tm_lock) 1 0 3 0 10 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c:3369 (sx:arc_user_evicts_lock) 0 0 6 0 10 0 0 0 0 /usr/src/sys/kern/vfs_bio.c:713 (rw:needsbuffer lock) 0 0 5 0 10 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/trim_map.c:455 (sx:tm->tm_lock) 0 0 4 0 10 0 0 0 0 /usr/src/sys/kern/kern_sx.c:195 (sx:spa->spa_trim_lock) 0 0 4 0 10 0 0 0 0 /usr/src/sys/kern/kern_sx.c:195 (sx:arc_user_evicts_lock) 0 0 4 0 10 0 0 0 0 /usr/src/sys/fs/nfsserver/nfs_nfsdstate.c:1075 (sleep mutex:nfs_state_mutex) 0 0 3 0 10 0 0 0 0 /usr/src/sys/netipsec/key.c:4424 (sleep mutex:acqtree) 0 0 3 0 10 0 0 0 0 /usr/src/sys/netipsec/key.c:4298 (sleep mutex:sahtree) 0 0 3 0 10 0 0 0 0 /usr/src/sys/kern/vfs_bio.c:763 (sleep mutex:runningbufspace lock) 0 0 3 0 10 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c:7012 (sx:l2arc_dev_mtx) 0 0 2 0 10 0 0 0 0 /usr/src/sys/netipsec/key.c:4442 (sleep mutex:spacqtree) 0 0 2 0 10 0 0 0 0 /usr/src/sys/netinet6/nd6.c:913 (rw:nd6) 0 0 2 0 10 0 0 0 0 /usr/src/sys/kern/vfs_bio.c:441 (sleep mutex:dirty buf lock) 0 0 1 0 10 0 0 0 0 /usr/src/sys/vm/vnode_pager.c:121 (rw:vm object) 0 0 1 0 10 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 16) 5 0 34 0 9 3 0 0 0 /usr/src/sys/kern/sys_generic.c:1833 (sleep mutex:sellck) 3 0 20 0 9 2 0 0 0 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 22) 1 0 4 0 9 0 0 0 0 /usr/src/sys/kern/tty_pts.c:121 (sleep mutex:ttymtx) 0 0 2 0 9 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 9) 32 0 77 0 8 9 0 0 0 /usr/src/sys/vm/vm_map.c:1458 (sx:vm map (user)) 9 0 18 0 8 2 0 0 0 /usr/src/sys/kern/kern_mutex.c:190 (sleep mutex:CAM device lock) 8 5 22 5 8 2 0 0 1 /usr/src/sys/kern/kern_sx.c:195 (sx:tx->tx_sync_lock) 7 0 20 0 8 2 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:139 (sx:db->db_mtx) 4 0 18 0 8 2 0 0 0 /usr/src/sys/kern/imgact_elf.c:480 (sx:vm map (user)) 1 0 8 0 8 1 0 0 0 /usr/src/sys/kern/subr_turnstile.c:210 (spin mutex:sched lock 0) 1 0 7 0 8 0 0 0 0 /usr/src/sys/cam/scsi/scsi_enc.c:786 (sleep mutex:CAM device lock) 1 0 3 0 8 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:714 (sx:dn->dn_struct_rwlock) 0 0 3 0 8 0 0 0 0 /usr/src/sys/cam/scsi/scsi_enc_ses.c:1009 (sleep mutex:CAM device lock) 0 0 1 0 8 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:2006 (sleep mutex:Syncer mtx) 0 0 1 0 8 0 0 0 0 /usr/src/sys/kern/sys_generic.c:777 (sx:filedesc structure) 0 0 1 0 8 0 0 0 0 /usr/src/sys/kern/kern_exit.c:332 (sleep mutex:process_exit) 0 0 0 0 8 0 0 0 0 /usr/src/sys/vm/vnode_pager.c:225 (rw:vm object) 0 0 0 0 8 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:527 (sx:zio->io_lock) 0 0 0 0 8 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dnode.c:1230 (sx:dn->dn_mtx) 4 0 18 0 7 2 0 0 0 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 20) 4 0 18 0 7 2 0 0 0 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 15) 0 0 0 0 7 0 0 0 0 /usr/src/sys/vm/vm_object.c:2080 (rw:vm object) 8 0 23 0 6 3 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:3708 (rw:pmap pv list) 5 0 11 0 6 1 0 0 0 /usr/src/sys/dev/usb/usb_request.c:493 (sleep mutex:ehci1) 2 0 8 0 6 1 0 0 0 /usr/src/sys/dev/usb/usb_request.c:493 (sleep mutex:ehci0) 2 0 8 0 6 1 0 0 0 /usr/src/sys/dev/usb/controller/ehci.c:3900 (sleep mutex:ehci1) 2 0 8 0 6 1 0 0 0 /usr/src/sys/dev/usb/controller/ehci.c:3900 (sleep mutex:ehci0) 1 0 8 0 6 1 0 0 0 /usr/src/sys/kern/vfs_vnops.c:1343 (lockmgr:zfs) 1 0 4 0 6 0 0 0 0 /usr/src/sys/kern/vfs_vnops.c:442 (lockmgr:zfs) 0 1 1 1 6 0 0 0 2 /usr/src/sys/kern/subr_turnstile.c:730 (spin mutex:sched lock 0) 0 0 1 0 6 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 8) 0 0 1 0 6 0 0 0 0 /usr/src/sys/kern/kern_descrip.c:1813 (sx:filedesc structure) 0 0 1 0 6 0 0 0 0 /usr/src/sys/kern/kern_descrip.c:1229 (sx:filedesc structure) 0 0 0 0 6 0 0 0 0 /usr/src/sys/vm/vm_reserv.c:1096 (sleep mutex:vm page free queue) 0 0 0 0 6 0 0 0 0 /usr/src/sys/kern/vfs_vnops.c:709 (sleep mutex:sleep mtxpool) 0 0 0 0 6 0 0 0 0 /usr/src/sys/kern/vfs_vnops.c:677 (sleep mutex:sleep mtxpool) 0 0 0 0 6 0 0 0 0 /usr/src/sys/kern/vfs_mount.c:443 (sleep mutex:struct mount mtx) 0 0 0 0 6 0 0 0 0 /usr/src/sys/kern/vfs_mount.c:433 (sleep mutex:struct mount mtx) 0 0 0 0 6 0 0 0 0 /usr/src/sys/kern/kern_proc.c:366 (sleep mutex:process group) 0 0 0 0 6 0 0 0 0 /usr/src/sys/kern/kern_exec.c:1079 (sleep mutex:process_exec) 0 0 0 0 6 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/bpobj.c:550 (sx:bpo->bpo_lock) 440 0 1880 0 5 376 0 0 0 /usr/src/sys/kern/kern_sysctl.c:1971 (sx:sysctl mem) 10 0 44 0 5 8 0 0 0 /usr/src/sys/vm/vm_map.c:2335 (sx:vm map (user)) 7 0 30 0 5 6 0 0 0 /usr/src/sys/vm/vm_object.c:2213 (rw:vm object) 4 0 10 0 5 2 0 0 0 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 13) 3 0 13 0 5 2 0 0 0 /usr/src/sys/vm/vm_map.c:2540 (sx:vm map (user)) 2 0 7 0 5 1 0 0 0 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 1) 1 3 5 4 5 1 0 0 3 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 13) 0 0 2 0 5 0 0 0 0 /usr/src/sys/vm/vm_map.c:2642 (sx:vm map (user)) 0 0 2 0 5 0 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:4919 (sleep mutex:pmap) 0 0 1 0 5 0 0 0 0 /usr/src/sys/vm/uma_core.c:2371 (sleep mutex:vmem btag) 0 0 1 0 5 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:730 (spin mutex:sched lock 4) 0 0 1 0 5 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 0) 62 0 239 0 4 59 0 0 0 /usr/src/sys/amd64/amd64/pmap.c:5435 (sleep mutex:pmap) 38 0 91 0 4 22 0 0 0 /usr/src/sys/kern/vfs_vnops.c:786 (lockmgr:zfs) 26 0 59 0 4 14 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu.c:411 (sx:dn->dn_struct_rwlock) 18 0 45 0 4 11 0 0 0 /usr/src/sys/vm/vm_map.c:3075 (sx:vm map (user)) 17 0 46 0 4 11 0 0 0 /usr/src/sys/cam/scsi/scsi_enc.c:759 (sleep mutex:CAM device lock) 13 0 36 0 4 9 0 0 0 /usr/src/sys/kern/vfs_subr.c:2072 (lockmgr:syncer) 10 0 28 0 4 7 0 0 0 /usr/src/sys/dev/mps/mpsvar.h:582 (sleep mutex:MPT2SAS lock) 9 0 21 0 4 5 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu.c:146 (sx:dn->dn_struct_rwlock) 8 0 21 0 4 5 0 0 0 /usr/src/sys/cam/cam_xpt.c:3330 (sleep mutex:MPT2SAS lock) 8 0 17 0 4 4 0 0 0 /usr/src/sys/kern/kern_exit.c:1146 (sx:proctree) 5 0 11 0 4 2 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dnode.c:1114 (sx:dn->dn_struct_rwlock) 4 0 8 0 4 2 0 0 0 /usr/src/sys/kern/kern_prot.c:405 (sx:proctree) 4 0 8 0 4 2 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c:2251 (sx:buf_hash_table.ht_locks[i].ht_lock) 3 0 7 0 4 1 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c:2245 (sx:buf->b_evict_lock) 2 1 5 1 4 1 0 0 1 /usr/src/sys/kern/kern_exit.c:920 (sleep mutex:process lock) 2 0 7 0 4 1 0 0 0 /usr/src/sys/kern/vfs_subr.c:2088 (sleep mutex:Syncer mtx) 2 0 7 0 4 1 0 0 0 /usr/src/sys/cam/cam_xpt.c:5264 (sleep mutex:CAM device lock) 2 0 6 0 4 1 0 0 0 /usr/src/sys/kern/vfs_subr.c:2661 (lockmgr:zfs) 1 0 3 0 4 0 0 0 1 /usr/src/sys/netinet/tcp_input.c:3053 (sleep mutex:so_rcv) 1 0 3 0 4 0 0 0 0 /usr/src/sys/vm/vm_map.c:1968 (sx:vm map (user)) 1 0 3 0 4 0 0 0 0 /usr/src/sys/vm/uma_core.c:1022 (sleep mutex:vmem btag) 1 0 3 0 4 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:4145 (rw:bufobj interlock) 1 0 3 0 4 0 0 0 0 /usr/src/sys/kern/kern_exec.c:981 (rw:vm object) 1 0 3 0 4 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:199 (sx:db->db_mtx) 1 0 3 0 4 0 0 0 0 /usr/src/sys/cam/cam_xpt.c:2526 (sleep mutex:CAM queue lock) 1 0 2 0 4 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:592 (sx:db->db_mtx) 1 0 2 0 4 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:1913 (sx:dn->dn_dbufs_mtx) 1 0 2 0 4 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c:2634 (sx:buf_hash_table.ht_locks[i].ht_lock) 1 0 1 0 4 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/multilist.c:148 (sx:mls->mls_lock) 0 0 3 0 4 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:2056 (sleep mutex:vnode interlock) 0 0 3 0 4 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:895 (spin mutex:sched lock 10) 0 0 3 0 4 0 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 1) 0 0 2 0 4 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:5318 (sleep mutex:vnode_free_list) 0 0 2 0 4 0 0 0 0 /usr/src/sys/kern/tty.c:1728 (sleep mutex:ttymtx) 0 0 2 0 4 0 0 0 0 /usr/src/sys/kern/tty.c:1719 (sx:proctree) 0 0 2 0 4 0 0 0 0 /usr/src/sys/kern/kern_synch.c:570 (spin mutex:sched lock 0) 0 0 1 0 4 0 0 0 0 /usr/src/sys/vm/vm_glue.c:237 (rw:vm object) 0 0 1 0 4 0 0 0 0 /usr/src/sys/vm/uma_core.c:2206 (sleep mutex:sackhole) 0 0 1 0 4 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:2662 (sleep mutex:vnode interlock) 0 0 1 0 4 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:2076 (rw:bufobj interlock) 0 0 1 0 4 0 0 0 0 /usr/src/sys/kern/subr_sleepqueue.c:680 (spin mutex:sched lock 1) 0 0 1 0 4 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 10) 0 0 1 0 4 0 0 0 0 /usr/src/sys/kern/kern_umtx.c:4470 (sleep mutex:umtx lock) 0 0 1 0 4 0 0 0 0 /usr/src/sys/kern/kern_rangelock.c:219 (sleep mutex:vnode interlock) 0 0 1 0 4 0 0 0 0 /usr/src/sys/kern/kern_proc.c:630 (sleep mutex:process group) 0 0 1 0 4 0 0 0 0 /usr/src/sys/kern/kern_mutex.c:190 (sleep mutex:CAM doneq) 0 0 1 0 4 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:488 (sx:zio->io_lock) 0 0 1 0 4 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:464 (sx:zio->io_lock) 0 0 1 0 4 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_rlock.c:538 (sx:zp->z_range_lock) 0 0 1 0 4 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_rlock.c:444 (sx:zp->z_range_lock) 0 0 1 0 4 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/multilist.c:184 (sx:mls->mls_lock) 0 0 1 0 4 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:187 (sx:h->hash_mutexes[i]) 0 0 1 0 4 0 0 0 0 /usr/src/sys/cam/cam_xpt.c:5306 (sleep mutex:CAM doneq) 0 0 1 0 4 0 0 0 0 /usr/src/sys/cam/cam_xpt.c:5227 (sleep mutex:CAM queue lock) 0 0 1 0 4 0 0 0 0 /usr/src/sys/cam/cam_xpt.c:4475 (sleep mutex:CAM doneq) 0 0 0 0 4 0 0 0 0 /usr/src/sys/vm/vm_glue.c:287 (sleep mutex:vm page) 0 0 0 0 4 0 0 0 0 /usr/src/sys/vm/vm_glue.c:251 (sleep mutex:vm page) 0 0 0 0 4 0 0 0 0 /usr/src/sys/vm/uma_core.c:2206 (sleep mutex:vmem btag) 0 0 0 0 4 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:578 (sleep mutex:struct mount mtx) 0 0 0 0 4 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:533 (sleep mutex:struct mount mtx) 0 0 0 0 4 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:5312 (sleep mutex:struct mount mtx) 0 0 0 0 4 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:5241 (sleep mutex:struct mount mtx) 0 0 0 0 4 0 0 0 0 /usr/src/sys/kern/tty.c:388 (sleep mutex:process lock) 0 0 0 0 4 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 9) 0 0 0 0 4 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 5) 0 0 0 0 4 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 3) 0 0 0 0 4 0 0 0 0 /usr/src/sys/kern/kern_rangelock.c:164 (sleep mutex:vnode interlock) 0 0 0 0 4 0 0 0 0 /usr/src/sys/kern/kern_prot.c:1233 (sleep mutex:process lock) 0 0 0 0 4 0 0 0 0 /usr/src/sys/kern/kern_exec.c:1050 (sleep mutex:vm page) 0 0 0 0 4 0 0 0 0 /usr/src/sys/kern/kern_exec.c:1028 (sleep mutex:vm page) 0 0 0 0 4 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:489 (sx:zio->io_lock) 0 0 0 0 4 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:465 (sx:zio->io_lock) 0 0 0 0 4 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dnode.c:1778 (sx:dn->dn_mtx) 0 0 0 0 4 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dnode.c:1173 (sx:dn->dn_mtx) 0 0 0 0 4 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu.c:479 (sx:db->db_mtx) 0 0 0 0 4 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:228 (sx:h->hash_mutexes[i]) 0 0 0 0 4 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:1979 (sx:dn->dn_dbufs_mtx) 0 0 0 0 4 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c:2677 (sx:buf->b_evict_lock) 0 0 0 0 4 0 0 0 0 /usr/src/sys/cam/cam_xpt.c:3335 (sleep mutex:CAM queue lock) 5 0 12 0 3 4 0 0 0 /usr/src/sys/dev/usb/usb_hub.c:2276 (sleep mutex:ehci0) 4 0 9 0 3 3 0 0 0 /usr/src/sys/dev/usb/usb_hub.c:2276 (sleep mutex:ehci1) 3 0 5 0 3 1 0 0 0 /usr/src/sys/vm/vm_map.c:1505 (sleep mutex:vm map (system)) 2 0 5 0 3 1 0 0 0 /usr/src/sys/dev/usb/usb_hub.c:2471 (sleep mutex:ehci1) 2 0 5 0 3 1 0 0 0 /usr/src/sys/dev/usb/usb_hub.c:2471 (sleep mutex:ehci0) 2 0 4 0 3 1 0 0 0 /usr/src/sys/kern/subr_vmem.c:854 (sleep mutex:kernel arena) 1 0 2 0 3 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:895 (spin mutex:sched lock 4) 1 0 2 0 3 0 0 0 0 /usr/src/sys/dev/usb/controller/usb_controller.c:413 (sleep mutex:ehci1) 1 0 2 0 3 0 0 0 0 /usr/src/sys/dev/usb/controller/usb_controller.c:413 (sleep mutex:ehci0) 0 0 1 0 3 0 0 0 0 /usr/src/sys/netinet/ip_input.c:690 (rw:if_addr_lock) 0 0 1 0 3 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:2223 (spin mutex:sched lock 8) 0 0 1 0 3 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:2223 (spin mutex:sched lock 5) 0 0 1 0 3 0 0 0 0 /usr/src/sys/dev/usb/controller/usb_controller.c:378 (sleep mutex:ehci0) 0 0 0 0 3 0 0 0 1 /usr/src/sys/vm/uma_core.c:2208 (sleep mutex:mbuf_jumbo_9k) 0 0 0 0 3 0 0 0 0 /usr/src/sys/vm/uma_core.c:2724 (sleep mutex:sackhole) 0 0 0 0 3 0 0 0 0 /usr/src/sys/vm/uma_core.c:2371 (sleep mutex:mbuf_jumbo_9k) 0 0 0 0 3 0 0 0 0 /usr/src/sys/vm/uma_core.c:2259 (sleep mutex:vmem btag) 0 0 0 0 3 0 0 0 0 /usr/src/sys/vm/uma_core.c:2259 (sleep mutex:mbuf_jumbo_9k) 0 0 0 0 3 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:730 (spin mutex:sched lock 6) 0 0 0 0 3 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 4) 0 0 0 0 3 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 11) 0 0 0 0 3 0 0 0 0 /usr/src/sys/dev/usb/controller/usb_controller.c:691 (sleep mutex:ehci1) 0 0 0 0 3 0 0 0 0 /usr/src/sys/dev/usb/controller/usb_controller.c:691 (sleep mutex:ehci0) 0 0 0 0 3 0 0 0 0 /usr/src/sys/dev/usb/controller/usb_controller.c:378 (sleep mutex:ehci1) 130 0 253 0 2 126 0 0 0 /usr/src/sys/vm/vm_map.c:3311 (sx:vm map (user)) 130 0 252 0 2 126 0 0 0 /usr/src/sys/vm/vm_map.c:3315 (sx:vm map (user)) 24 0 37 0 2 18 0 0 0 /usr/src/sys/cam/scsi/scsi_enc_ses.c:1894 (sx:enccache) 16 0 24 0 2 12 0 0 0 /usr/src/sys/kern/kern_exit.c:497 (sleep mutex:process lock) 15 0 29 0 2 14 0 0 0 /usr/src/sys/kern/imgact_elf.c:899 (lockmgr:zfs) 12 0 23 0 2 11 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zap_micro.c:515 (sx:zap->zap_rwlock) 12 0 22 0 2 11 0 0 0 /usr/src/sys/vm/vm_kern.c:476 (sx:vm map (user)) 12 0 18 0 2 9 0 0 0 /usr/src/sys/kern/kern_exit.c:539 (sleep mutex:process lock) 9 0 18 0 2 9 0 0 0 /usr/src/sys/vm/vm_kern.c:343 (rw:kmem vm object) 6 0 13 0 2 6 0 0 0 /usr/src/sys/x86/x86/mp_x86.c:1402 (spin mutex:smp rendezvous) 6 0 8 0 2 4 0 0 0 /usr/src/sys/kern/kern_exit.c:446 (sx:proctree) 4 0 7 0 2 3 0 0 0 /usr/src/sys/kern/kern_exec.c:729 (lockmgr:zfs) 3 0 7 0 2 3 0 0 0 /usr/src/sys/kern/kern_fork.c:469 (sleep mutex:process lock) 3 0 7 0 2 3 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/txg.c:520 (sx:tx->tx_sync_lock) 3 0 6 0 2 3 0 0 0 /usr/src/sys/kern/kern_fork.c:470 (sleep mutex:process lock) 3 0 6 0 2 3 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/txg.c:566 (sx:tx->tx_sync_lock) 2 0 4 0 2 2 0 0 0 /usr/src/sys/kern/kern_exit.c:594 (spin mutex:process slock) 2 0 4 0 2 2 0 0 0 /usr/src/sys/kern/kern_descrip.c:1855 (sx:filedesc structure) 2 0 3 0 2 1 0 0 0 /usr/src/sys/net/bpf.c:2269 (rw:bpf interface lock) 2 0 3 0 2 1 0 0 0 /usr/src/sys/kern/subr_vmem.c:1141 (sleep mutex:kmem arena) 2 0 3 0 2 1 0 0 0 /usr/src/sys/kern/kern_exit.c:216 (sleep mutex:process lock) 2 0 2 0 2 1 0 0 0 /usr/src/sys/kern/kern_proc.c:711 (sx:proctree) 1 2 1 2 2 0 1 0 1 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 19) 1 0 3 0 2 1 0 0 1 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 17) 1 0 3 0 2 1 0 0 0 /usr/src/sys/vm/vm_object.c:495 (lockmgr:zfs) 1 0 3 0 2 1 0 0 0 /usr/src/sys/kern/kern_fork.c:958 (sx:allproc) 1 0 3 0 2 1 0 0 0 /usr/src/sys/kern/kern_exec.c:731 (sleep mutex:process lock) 1 0 2 0 2 1 0 0 0 /usr/src/sys/vm/vm_kern.c:445 (sx:vm map (user)) 1 0 2 0 2 1 0 0 0 /usr/src/sys/kern/vfs_subr.c:2742 (sleep mutex:vnode interlock) 1 0 2 0 2 1 0 0 0 /usr/src/sys/kern/kern_fork.c:589 (sleep mutex:process lock) 1 0 2 0 2 1 0 0 0 /usr/src/sys/kern/kern_fork.c:587 (sx:proctree) 1 0 2 0 2 1 0 0 0 /usr/src/sys/kern/kern_fork.c:404 (sleep mutex:process lock) 1 0 2 0 2 1 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zap.c:523 (sx:l->l_rwlock) 1 0 1 0 2 0 0 0 0 /usr/src/sys/vm/vm_object.c:497 (rw:vm object) 1 0 1 0 2 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:895 (spin mutex:sched lock 20) 1 0 1 0 2 0 0 0 0 /usr/src/sys/kern/sched_ule.c:901 (spin mutex:sched lock 21) 1 0 1 0 2 0 0 0 0 /usr/src/sys/kern/kern_thread.c:533 (spin mutex:pstatl) 1 0 1 0 2 0 0 0 0 /usr/src/sys/kern/kern_fork.c:957 (sx:proctree) 1 0 1 0 2 0 0 0 0 /usr/src/sys/kern/kern_exit.c:843 (sleep mutex:process lock) 0 2 0 3 2 0 1 0 2 /usr/src/sys/vm/uma_core.c:2726 (sleep mutex:mbuf_jumbo_9k) 0 1 1 2 2 0 1 0 2 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 15) 0 1 0 2 2 0 1 0 2 /usr/src/sys/kern/subr_turnstile.c:210 (spin mutex:sched lock 10) 0 1 0 1 2 0 0 0 2 /usr/src/sys/kern/subr_turnstile.c:210 (spin mutex:sched lock 20) 0 0 1 0 2 0 0 0 0 /usr/src/sys/vm/vm_map.c:3471 (sx:vm map (user)) 0 0 1 0 2 0 0 0 0 /usr/src/sys/vm/uma_core.c:1022 (sleep mutex:RADIX NODE) 0 0 1 0 2 0 0 0 0 /usr/src/sys/modules/mlxen/../../ofed/drivers/net/mlx4/en_rx.c:724 (spin mutex:sched lock 9) 0 0 1 0 2 0 0 0 0 /usr/src/sys/libkern/arc4random.c:135 (sleep mutex:arc4_mtx) 0 0 1 0 2 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:895 (spin mutex:sched lock 9) 0 0 1 0 2 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:895 (spin mutex:sched lock 8) 0 0 1 0 2 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:895 (spin mutex:sched lock 17) 0 0 1 0 2 0 0 0 0 /usr/src/sys/kern/subr_taskqueue.c:467 (spin mutex:mlx4_en_que) 0 0 1 0 2 0 0 0 0 /usr/src/sys/kern/subr_eventhandler.c:254 (sleep mutex:process_exit) 0 0 1 0 2 0 0 0 0 /usr/src/sys/kern/kern_synch.c:309 (spin mutex:mlx4_en_que) 0 0 1 0 2 0 0 0 0 /usr/src/sys/kern/kern_synch.c:309 (spin mutex:mlx4_en_que) 0 0 1 0 2 0 0 0 0 /usr/src/sys/kern/kern_sig.c:968 (sleep mutex:sigacts) 0 0 1 0 2 0 0 0 0 /usr/src/sys/kern/kern_sig.c:3632 (sleep mutex:sigacts) 0 0 1 0 2 0 0 0 0 /usr/src/sys/kern/kern_proc.c:552 (sleep mutex:process group) 0 0 1 0 2 0 0 0 0 /usr/src/sys/kern/kern_proc.c:382 (sx:allproc) 0 0 1 0 2 0 0 0 0 /usr/src/sys/kern/kern_ktrace.c:575 (sleep mutex:process lock) 0 0 1 0 2 0 0 0 0 /usr/src/sys/kern/kern_fork.c:702 (sleep mutex:process lock) 0 0 1 0 2 0 0 0 0 /usr/src/sys/kern/kern_fork.c:701 (sleep mutex:process lock) 0 0 1 0 2 0 0 0 0 /usr/src/sys/kern/kern_fork.c:590 (sleep mutex:process lock) 0 0 1 0 2 0 0 0 0 /usr/src/sys/kern/kern_fork.c:588 (sleep mutex:process group) 0 0 1 0 2 0 0 0 0 /usr/src/sys/kern/kern_exit.c:796 (sleep mutex:process lock) 0 0 1 0 2 0 0 0 0 /usr/src/sys/kern/kern_exit.c:601 (spin mutex:pstatl) 0 0 1 0 2 0 0 0 0 /usr/src/sys/kern/kern_descrip.c:2305 (sx:filedesc structure) 0 0 1 0 2 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_errlog.c:349 (sx:spa->spa_errlist_lock) 0 0 1 0 2 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c:6260 (sx:spa->spa_async_lock) 0 0 0 0 2 0 0 0 2 /usr/src/sys/kern/subr_turnstile.c:210 (spin mutex:sched lock 4) 0 0 0 0 2 0 0 0 1 /usr/src/sys/kern/subr_turnstile.c:210 (spin mutex:sched lock 17) 0 0 0 0 2 0 0 0 0 /usr/src/sys/vm/vm_object.c:1774 (rw:vm object) 0 0 0 0 2 0 0 0 0 /usr/src/sys/vm/vm_map.c:3985 (sleep mutex:vm map (system)) 0 0 0 0 2 0 0 0 0 /usr/src/sys/vm/vm_map.c:3368 (rw:vm object) 0 0 0 0 2 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:mbuf) 0 0 0 0 2 0 0 0 0 /usr/src/sys/vm/uma_core.c:2724 (sleep mutex:4096) 0 0 0 0 2 0 0 0 0 /usr/src/sys/vm/uma_core.c:2724 (sleep mutex:16384) 0 0 0 0 2 0 0 0 0 /usr/src/sys/vm/uma_core.c:2371 (sleep mutex:UMA Slabs) 0 0 0 0 2 0 0 0 0 /usr/src/sys/vm/uma_core.c:2206 (sleep mutex:16384) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/vfs_vnops.c:2468 (sx:pmc-sx) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:2823 (sleep mutex:vnode_free_list) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:2757 (sleep mutex:vnode_free_list) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:2223 (spin mutex:sched lock 23) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/sysv_sem.c:1399 (sleep mutex:semu) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/subr_taskqueue.c:467 (spin mutex:mlx4_en_que) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/subr_taskqueue.c:284 (spin mutex:mlx4_en_que) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/subr_taskqueue.c:284 (spin mutex:mlx4_en_que) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/subr_eventhandler.c:254 (sleep mutex:process_exec) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/subr_eventhandler.c:254 (sleep mutex:process_dtor) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/subr_eventhandler.c:254 (sleep mutex:process_ctor) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 7) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 6) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_umtx.c:4350 (sleep mutex:process lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_umtx.c:4340 (sleep mutex:process lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_thread.c:1200 (rw:tidhash) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_thread.c:1192 (rw:tidhash) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_sig.c:2136 (sleep mutex:sigacts) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_proc.c:775 (sleep mutex:process lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_proc.c:702 (sleep mutex:process lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_proc.c:615 (sleep mutex:ttymtx) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_proc.c:607 (sleep mutex:process group) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_proc.c:577 (sleep mutex:process lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_proc.c:576 (sleep mutex:process group) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_proc.c:554 (sleep mutex:process lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_proc.c:553 (sleep mutex:process group) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_proc.c:480 (sleep mutex:process group) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_proc.c:224 (sleep mutex:process_dtor) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_proc.c:197 (sleep mutex:process_ctor) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_proc.c:1922 (sleep mutex:process lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_proc.c:1030 (spin mutex:sched lock 4) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_proc.c:1030 (spin mutex:sched lock 22) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_proc.c:1030 (spin mutex:sched lock 17) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_proc.c:1030 (spin mutex:sched lock 16) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_proc.c:1030 (spin mutex:sched lock 10) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_ktrace.c:576 (sleep mutex:ktrace) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_jail.c:2754 (sleep mutex:jail mutex) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_jail.c:2743 (sleep mutex:jail mutex) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_fork.c:774 (sleep mutex:process lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_fork.c:704 (spin mutex:process slock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_fork.c:598 (sleep mutex:session) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_fork.c:405 (sleep mutex:process lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_exit.c:856 (sleep mutex:process lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_exit.c:835 (sleep mutex:process lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_exit.c:829 (sx:allproc) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_exit.c:540 (sleep mutex:sigacts) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_exit.c:425 (sx:allproc) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_exit.c:338 (sleep mutex:process lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_exit.c:1275 (sleep mutex:process lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_exit.c:1161 (spin mutex:process slock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_exit.c:1160 (sleep mutex:process lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_exit.c:1055 (spin mutex:process slock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_exec.c:907 (sleep mutex:process lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_exec.c:394 (sleep mutex:process lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_event.c:2133 (sleep mutex:process lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_descrip.c:2183 (sx:filedesc structure) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/kern_descrip.c:2176 (sleep mutex:process lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/imgact_elf.c:972 (sleep mutex:process lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/kern/imgact_elf.c:1034 (lockmgr:zfs) 0 0 0 0 2 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c:6273 (sx:spa->spa_async_lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/rrwlock.c:216 (sx:rrl->rr_lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/bplist.c:67 (sx:bpl->bpl_lock) 0 0 0 0 2 0 0 0 0 /usr/src/sys/amd64/amd64/vm_machdep.c:305 (sleep mutex:descriptor tables) 0 0 0 0 2 0 0 0 0 /usr/src/sys/amd64/amd64/vm_machdep.c:250 (sleep mutex:descriptor tables) 0 0 0 0 2 0 0 0 0 /usr/src/sys/amd64/amd64/machdep.c:592 (sleep mutex:descriptor tables) 0 0 0 0 2 0 0 0 0 /usr/src/sys/amd64/amd64/../../kern/subr_syscall.c:202 (sleep mutex:process lock) 94 0 94 0 1 94 0 0 0 /usr/src/sys/vm/uma_core.c:1747 (rw:UMA lock) 19 0 19 0 1 19 0 0 0 /usr/src/sys/vm/vm_glue.c:754 (sx:allproc) 3 0 3 0 1 3 0 0 0 /usr/src/sys/kern/subr_vmem.c:732 (sleep mutex:vmem list lock) 2 0 2 0 1 2 0 0 0 /usr/src/sys/net/route.c:983 (rw:rib head lock) 1 0 1 0 1 1 0 0 0 /usr/src/sys/net/route.c:457 (rw:rib head lock) 1 0 1 0 1 1 0 0 0 /usr/src/sys/kern/subr_turnstile.c:895 (spin mutex:sched lock 18) 1 0 1 0 1 1 0 0 0 /usr/src/sys/kern/subr_taskqueue.c:467 (spin mutex:mlx4_en_que) 1 0 1 0 1 1 0 0 0 /usr/src/sys/kern/subr_taskqueue.c:467 (sleep mutex:thread) 1 0 1 0 1 1 0 0 0 /usr/src/sys/kern/kern_thread.c:534 (spin mutex:sched lock 23) 1 0 1 0 1 1 0 0 0 /usr/src/sys/kern/kern_thread.c:534 (spin mutex:sched lock 22) 1 0 1 0 1 1 0 0 0 /usr/src/sys/kern/kern_fork.c:764 (spin mutex:sched lock 5) 1 0 1 0 1 1 0 0 0 /usr/src/sys/kern/kern_fork.c:764 (spin mutex:sched lock 20) 0 1 0 1 1 0 1 0 1 /usr/src/sys/kern/sched_ule.c:2175 (spin mutex:sched lock 5) 0 0 0 0 1 0 0 0 1 /usr/src/sys/kern/subr_turnstile.c:210 (spin mutex:sched lock 7) 0 0 0 0 1 0 0 0 1 /usr/src/sys/kern/sched_ule.c:904 (spin mutex:sched lock 21) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_link_cache) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_98304) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_917504) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_8388608) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_81920) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_8192) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_786432) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_7340032) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_7168) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_655360) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_65536) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_6291456) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_6144) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_57344) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_5242880) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_524288) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_5120) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_512) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_49152) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_458752) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_4194304) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_40960) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_4096) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_393216) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_3670016) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_3584) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_327680) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_32768) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_3145728) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_3072) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_28672) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_2621440) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_262144) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_2560) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_24576) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_229376) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_2097152) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_20480) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_2048) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_196608) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_1835008) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_16777216) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_163840) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_16384) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_1572864) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_1536) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_14680064) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_14336) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_1310720) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_131072) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_12582912) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_12288) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_114688) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_10485760) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_1048576) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_10240) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_data_buf_1024) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_cache) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_98304) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_917504) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_8388608) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_81920) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_8192) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_786432) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_7340032) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_7168) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_655360) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_65536) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_6291456) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_6144) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_57344) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_5242880) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_524288) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_5120) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_512) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_49152) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_458752) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_4194304) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_40960) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_4096) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_393216) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_3670016) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_3584) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_327680) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_32768) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_3145728) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_3072) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_28672) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_2621440) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_262144) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_2560) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_24576) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_229376) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_2097152) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_20480) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_2048) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_196608) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_1835008) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_16777216) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_163840) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_16384) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_1572864) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_1536) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_14680064) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_14336) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_1310720) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_131072) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_12582912) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_12288) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_114688) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_10485760) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_1048576) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_10240) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zio_buf_1024) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zil_lwb_cache) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:zfs_znode_cache) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:vtnet_tx_hdr) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:vmem btag) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:unpcb) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:umtx_shm) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:umtx pi) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:udplite_inpcb) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:udpcb) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:udp_inpcb) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:ttyoutq) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:ttyinq) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:tcptw) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:tcpreass) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:tcpcb) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:tcp_inpcb) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:taskq_zone) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:syncache) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:socket) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:selfd) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:sctp_stream_msg_out) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:sctp_readq) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:sctp_raddr) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:sctp_laddr) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:sctp_ep) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:sctp_chunk) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:sctp_asoc) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:sctp_asconf_ack) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:sctp_asconf) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:sackhole) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:sa_cache) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:rtentry) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:rl_entry) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:ripcb) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:range_seg_cache) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:procdesc) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:pipe) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:nvme_request) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:mt_zone) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:mlx4_cmd) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:mbuf_jumbo_page) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:mbuf_jumbo_9k) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:mbuf_jumbo_16k) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:mbuf_cluster) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:lz4_ctx) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:ksiginfo) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:itimer) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:ipq) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:hostcache) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:g_bio) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:filedesc0) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:fakepg) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:dnode_t) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:dmu_buf_impl_t) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:cryptop) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:cryptodesc) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:cpuset) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:audit_record) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:arc_buf_t) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:arc_buf_hdr_t_l2only) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:arc_buf_hdr_t_full) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:VNODEPOLL) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:VNODE) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:VMSPACE) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:VM OBJECT) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:UMA Zones) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:UMA Slabs) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:UMA Kegs) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:UMA Hash) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:TURNSTILE) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:THREAD) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:SWAPMETA) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:STS VFS Cache) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:SLEEPQUEUE) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:S VFS Cache) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:RADIX NODE) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:PROC) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:NCLNODE) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:NAMEI) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:Mountpoints) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:MAP) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:MAP ENTRY) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:MAC labels) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:LTS VFS Cache) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:L VFS Cache) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:KNOTE) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:KMAP ENTRY) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:Files) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:FPU_save_area) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:DMAR_MAP_ENTRY) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:DIRHASH) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:BUF TRIE) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:AIOP) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:AIOLIO) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:AIOL) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:AIOCB) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:AIO) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:8192) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:8 Bucket) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:65536) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:64) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:64 pcpu) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:64 Bucket) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:6 Bucket) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:512) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:4096) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:4 Bucket) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:32768) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:32) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:32 Bucket) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:256) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:256 Bucket) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:2048) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:16384) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:16) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:16 Bucket) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:128) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:128 Bucket) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:12 Bucket) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:480 (sleep mutex:1024) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:2838 (sleep mutex:32 Bucket) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:2724 (sleep mutex:selfd) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:2724 (sleep mutex:VMSPACE) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:2724 (sleep mutex:32768) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:2371 (sleep mutex:mbuf) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:2371 (sleep mutex:RADIX NODE) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:2371 (sleep mutex:65536) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:2371 (sleep mutex:32768) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:2259 (sleep mutex:mbuf) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:2259 (sleep mutex:RADIX NODE) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:2259 (sleep mutex:65536) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:2259 (sleep mutex:32768) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:2206 (sleep mutex:VM OBJECT) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:2206 (sleep mutex:RADIX NODE) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:2206 (sleep mutex:65536) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:2206 (sleep mutex:64 Bucket) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:2206 (sleep mutex:32768) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:2206 (sleep mutex:256) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:1022 (sleep mutex:65536) 0 0 0 0 1 0 0 0 0 /usr/src/sys/vm/uma_core.c:1022 (sleep mutex:32768) 0 0 0 0 1 0 0 0 0 /usr/src/sys/netinet/ip_output.c:712 (sleep mutex:rtentry) 0 0 0 0 1 0 0 0 0 /usr/src/sys/netinet/if_ether.c:609 (rw:if_afdata) 0 0 0 0 1 0 0 0 0 /usr/src/sys/net/route.c:461 (sleep mutex:rtentry) 0 0 0 0 1 0 0 0 0 /usr/src/sys/modules/mlxen/../../ofed/drivers/net/mlx4/en_rx.c:724 (spin mutex:sched lock 0) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:2223 (spin mutex:sched lock 9) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/vfs_subr.c:2223 (spin mutex:sched lock 16) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/subr_vmem.c:274 (sleep mutex:kernel arena) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:895 (spin mutex:sched lock 7) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:895 (spin mutex:sched lock 6) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:730 (spin mutex:sched lock 9) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:730 (spin mutex:sched lock 7) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:730 (spin mutex:sched lock 2) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:730 (spin mutex:sched lock 11) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/subr_turnstile.c:210 (spin mutex:sched lock 18) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/subr_taskqueue.c:467 (spin mutex:mlx4_en_que) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/subr_taskqueue.c:284 (spin mutex:mlx4_en_que) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/subr_taskqueue.c:284 (spin mutex:mlx4_en_que) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/subr_taskqueue.c:284 (sleep mutex:thread) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2754 (spin mutex:sched lock 9) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2754 (spin mutex:sched lock 14) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2226 (spin mutex:sched lock 2) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/sched_ule.c:2175 (spin mutex:sched lock 20) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/kern_umtx.c:4477 (spin mutex:sched lock 9) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/kern_umtx.c:4477 (spin mutex:sched lock 23) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/kern_umtx.c:4477 (spin mutex:sched lock 22) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/kern_umtx.c:4477 (spin mutex:sched lock 14) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/kern_thread.c:576 (spin mutex:sched lock 23) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/kern_thread.c:576 (spin mutex:sched lock 22) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/kern_synch.c:309 (spin mutex:mlx4_en_que) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/kern_synch.c:309 (spin mutex:mlx4_en_que) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/kern_mutex.c:190 (sleep mutex:thread) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/kern_fork.c:491 (spin mutex:sched lock 5) 0 0 0 0 1 0 0 0 0 /usr/src/sys/kern/kern_fork.c:491 (spin mutex:sched lock 20)