Loading snapshot checkpoint/run1/model-507...
Loaded in 0.000003 seconds
Loading dataset...
100%|████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00,  1.27it/s]
dataset has 34988665 tokens 1 chunks
Training...
save <function main.<locals>.save at 0x13970ca70>
main_command_generate_samples <function main.<locals>.generate_samples at 0x13970cb00>
main_command_validation <function main.<locals>.validation at 0x13970cc20>
2019-11-18T03:36:15.032873-08:00 [508 | 0.8467] Running opt_apply...
2019-11-18T03:36:21.593572-08:00 [508 | 7.4074 | 7.41s | 138.240438tokens/s] loss=11.0413 avg=11.0413 rate=0.0000200 step=0
2019-11-18T03:36:21.715312-08:00 [509 | 7.5291] Running opt_apply...
2019-11-18T03:36:24.526391-08:00 [509 | 10.3402 | 2.93s | 349.151433tokens/s] loss=8.6852 avg=9.8573 rate=0.0000200 step=1
2019-11-18T03:36:24.651931-08:00 [510 | 10.4658] Running opt_apply...
2019-11-18T03:36:27.289649-08:00 [510 | 13.1035 | 2.76s | 370.577149tokens/s] loss=10.5544 avg=10.0920 rate=0.0000200 step=2
2019-11-18T03:36:27.398254-08:00 [511 | 13.2121] Running opt_apply...
2019-11-18T03:36:30.086966-08:00 [511 | 15.9008 | 2.80s | 366.065176tokens/s] loss=10.6445 avg=10.2322 rate=0.0000200 step=3
2019-11-18T03:36:30.196077-08:00 [512 | 16.0099] Running opt_apply...
2019-11-18T03:36:32.770198-08:00 [512 | 18.5840 | 2.68s | 381.629173tokens/s] loss=10.1826 avg=10.2221 rate=0.0000200 step=4
2019-11-18T03:36:32.884888-08:00 [513 | 18.6987] Running opt_apply...
2019-11-18T03:36:35.490063-08:00 [513 | 21.3039 | 2.72s | 376.489296tokens/s] loss=9.5972 avg=10.1153 rate=0.0000200 step=5
2019-11-18T03:36:35.598292-08:00 [514 | 21.4121] Running opt_apply...
2019-11-18T03:36:38.160843-08:00 [514 | 23.9747 | 2.67s | 383.408327tokens/s] loss=9.9510 avg=10.0911 rate=0.0000200 step=6
2019-11-18T03:36:38.266506-08:00 [515 | 24.0803] Running opt_apply...
2019-11-18T03:36:40.851390-08:00 [515 | 26.6652 | 2.69s | 380.592025tokens/s] loss=7.4424 avg=9.7483 rate=0.0000200 step=7
2019-11-18T03:36:40.968085-08:00 [516 | 26.7819] Running opt_apply...
2019-11-18T03:36:43.530450-08:00 [516 | 29.3443 | 2.68s | 382.223618tokens/s] loss=8.9875 avg=9.6603 rate=0.0000200 step=8
2019-11-18T03:36:43.626938-08:00 [517 | 29.4408] Running opt_apply...
2019-11-18T03:36:46.175221-08:00 [517 | 31.9890 | 2.64s | 387.179344tokens/s] loss=9.1908 avg=9.6112 rate=0.0000200 step=9
2019-11-18T03:36:46.268683-08:00 [518 | 32.0825] Running opt_apply...
2019-11-18T03:36:48.841603-08:00 [518 | 34.6554 | 2.67s | 384.040745tokens/s] loss=8.8476 avg=9.5383 rate=0.0000200 step=10
2019-11-18T03:36:48.947400-08:00 [519 | 34.7612] Running opt_apply...
2019-11-18T03:36:51.521894-08:00 [519 | 37.3357 | 2.68s | 382.048214tokens/s] loss=8.5746 avg=9.4534 rate=0.0000200 step=11
2019-11-18T03:36:51.637079-08:00 [520 | 37.4509] Running opt_apply...
2019-11-18T03:36:54.255076-08:00 [520 | 40.0689 | 2.73s | 374.654732tokens/s] loss=8.5405 avg=9.3789 rate=0.0000200 step=12
2019-11-18T03:36:54.365059-08:00 [521 | 40.1789] Running opt_apply...
2019-11-18T03:36:56.941919-08:00 [521 | 42.7557 | 2.69s | 381.116133tokens/s] loss=8.7336 avg=9.3297 rate=0.0000200 step=13
2019-11-18T03:36:57.049716-08:00 [522 | 42.8635] Running opt_apply...
2019-11-18T03:36:59.682632-08:00 [522 | 45.4965 | 2.74s | 373.625810tokens/s] loss=6.7091 avg=9.1425 rate=0.0000200 step=14
2019-11-18T03:36:59.797679-08:00 [523 | 45.6115] Running opt_apply...
2019-11-18T03:37:02.344844-08:00 [523 | 48.1587 | 2.66s | 384.642561tokens/s] loss=8.4523 avg=9.0960 rate=0.0000200 step=15
2019-11-18T03:37:02.457937-08:00 [524 | 48.2718] Running opt_apply...
2019-11-18T03:37:05.011090-08:00 [524 | 50.8249 | 2.67s | 384.060423tokens/s] loss=8.3013 avg=9.0454 rate=0.0000200 step=16
2019-11-18T03:37:05.115975-08:00 [525 | 50.9298] Running opt_apply...
2019-11-18T03:37:07.682432-08:00 [525 | 53.4963 | 2.67s | 383.327945tokens/s] loss=8.4175 avg=9.0075 rate=0.0000200 step=17
2019-11-18T03:37:07.786870-08:00 [526 | 53.6007] Running opt_apply...
2019-11-18T03:37:10.434790-08:00 [526 | 56.2486 | 2.75s | 372.044630tokens/s] loss=8.3068 avg=8.9671 rate=0.0000200 step=18
2019-11-18T03:37:10.548506-08:00 [527 | 56.3623] Running opt_apply...
2019-11-18T03:37:13.097159-08:00 [527 | 58.9110 | 2.66s | 384.619723tokens/s] loss=8.0760 avg=8.9182 rate=0.0000200 step=19
2019-11-18T03:37:13.206600-08:00 [528 | 59.0204] Running opt_apply...
2019-11-18T03:37:15.741683-08:00 [528 | 61.5555 | 2.64s | 387.215227tokens/s] loss=7.6747 avg=8.8529 rate=0.0000200 step=20
2019-11-18T03:37:15.859349-08:00 [529 | 61.6732] Running opt_apply...
2019-11-18T03:37:18.423265-08:00 [529 | 64.2371 | 2.68s | 381.864278tokens/s] loss=8.2379 avg=8.8219 rate=0.0000200 step=21
2019-11-18T03:37:18.532822-08:00 [530 | 64.3466] Running opt_apply...
2019-11-18T03:37:21.085257-08:00 [530 | 66.8991 | 2.66s | 384.674324tokens/s] loss=7.8917 avg=8.7768 rate=0.0000200 step=22
2019-11-18T03:37:21.203174-08:00 [531 | 67.0170] Running opt_apply...
2019-11-18T03:37:23.762573-08:00 [531 | 69.5764 | 2.68s | 382.472330tokens/s] loss=7.9384 avg=8.7377 rate=0.0000200 step=23
2019-11-18T03:37:24.378795-08:00 [532 | 70.1926] Running opt_apply...
2019-11-18T03:37:27.033744-08:00 [532 | 72.8476 | 3.27s | 313.038035tokens/s] loss=7.8020 avg=8.6956 rate=0.0000200 step=24
2019-11-18T03:37:27.152248-08:00 [533 | 72.9661] Running opt_apply...
2019-11-18T03:37:29.732840-08:00 [533 | 75.5467 | 2.70s | 379.385853tokens/s] loss=7.2372 avg=8.6321 rate=0.0000200 step=25
2019-11-18T03:37:29.844601-08:00 [534 | 75.6584] Running opt_apply...
2019-11-18T03:37:32.446669-08:00 [534 | 78.2605 | 2.71s | 377.326941tokens/s] loss=7.4108 avg=8.5808 rate=0.0000200 step=26
2019-11-18T03:37:32.561832-08:00 [535 | 78.3757] Running opt_apply...
2019-11-18T03:37:35.182623-08:00 [535 | 80.9964 | 2.74s | 374.275292tokens/s] loss=6.1068 avg=8.4799 rate=0.0000200 step=27
2019-11-18T03:37:35.288313-08:00 [536 | 81.1021] Running opt_apply...
2019-11-18T03:37:37.912465-08:00 [536 | 83.7263 | 2.73s | 375.113161tokens/s] loss=6.0110 avg=8.3822 rate=0.0000200 step=28
2019-11-18T03:37:38.015503-08:00 [537 | 83.8293] Running opt_apply...
2019-11-18T03:37:40.572870-08:00 [537 | 86.3867 | 2.66s | 384.904228tokens/s] loss=7.2733 avg=8.3396 rate=0.0000200 step=29
2019-11-18T03:37:40.683816-08:00 [538 | 86.4976] Running opt_apply...
2019-11-18T03:37:43.253735-08:00 [538 | 89.0676 | 2.68s | 381.965990tokens/s] loss=6.0039 avg=8.2524 rate=0.0000200 step=30
2019-11-18T03:37:43.357420-08:00 [539 | 89.1712] Running opt_apply...
2019-11-18T03:37:45.970281-08:00 [539 | 91.7841 | 2.72s | 376.949250tokens/s] loss=8.1092 avg=8.2472 rate=0.0000200 step=31
2019-11-18T03:37:46.095767-08:00 [540 | 91.9096] Running opt_apply...
2019-11-18T03:37:48.662646-08:00 [540 | 94.4765 | 2.69s | 380.334502tokens/s] loss=6.0851 avg=8.1706 rate=0.0000200 step=32
2019-11-18T03:37:48.772313-08:00 [541 | 94.5861] Running opt_apply...
2019-11-18T03:37:51.353232-08:00 [541 | 97.1671 | 2.69s | 380.586494tokens/s] loss=5.8951 avg=8.0920 rate=0.0000200 step=33
2019-11-18T03:37:51.462434-08:00 [542 | 97.2763] Running opt_apply...
2019-11-18T03:37:54.323730-08:00 [542 | 100.1376 | 2.97s | 344.723222tokens/s] loss=6.9176 avg=8.0524 rate=0.0000200 step=34
2019-11-18T03:37:54.426223-08:00 [543 | 100.2400] Running opt_apply...
2019-11-18T03:37:57.132639-08:00 [543 | 102.9465 | 2.81s | 364.554607tokens/s] loss=7.9910 avg=8.0503 rate=0.0000200 step=35
2019-11-18T03:37:57.242705-08:00 [544 | 103.0565] Running opt_apply...
2019-11-18T03:37:59.824528-08:00 [544 | 105.6383 | 2.69s | 380.402042tokens/s] loss=6.0922 avg=7.9873 rate=0.0000200 step=36
2019-11-18T03:37:59.926980-08:00 [545 | 105.7408] Running opt_apply...
2019-11-18T03:38:02.474535-08:00 [545 | 108.2884 | 2.65s | 386.413898tokens/s] loss=6.7170 avg=7.9473 rate=0.0000200 step=37
2019-11-18T03:38:02.582412-08:00 [546 | 108.3962] Running opt_apply...
2019-11-18T03:38:05.174337-08:00 [546 | 110.9882 | 2.70s | 379.287085tokens/s] loss=8.3215 avg=7.9588 rate=0.0000200 step=38
2019-11-18T03:38:05.518988-08:00 [547 | 111.3328] Running opt_apply...
2019-11-18T03:38:08.154354-08:00 [547 | 113.9682 | 2.98s | 343.621960tokens/s] loss=8.2803 avg=7.9685 rate=0.0000200 step=39
2019-11-18T03:38:08.257710-08:00 [548 | 114.0715] Running opt_apply...
2019-11-18T03:38:10.804945-08:00 [548 | 116.6188 | 2.65s | 386.329403tokens/s] loss=7.2954 avg=7.9486 rate=0.0000200 step=40
2019-11-18T03:38:10.911366-08:00 [549 | 116.7252] Running opt_apply...
2019-11-18T03:38:13.517322-08:00 [549 | 119.3311 | 2.71s | 377.528630tokens/s] loss=6.2827 avg=7.9002 rate=0.0000200 step=41
2019-11-18T03:38:13.628093-08:00 [550 | 119.4419] Running opt_apply...
2019-11-18T03:38:16.178167-08:00 [550 | 121.9920 | 2.66s | 384.840286tokens/s] loss=6.8100 avg=7.8691 rate=0.0000200 step=42
2019-11-18T03:38:16.276088-08:00 [551 | 122.0899] Running opt_apply...
2019-11-18T03:38:18.825437-08:00 [551 | 124.6393 | 2.65s | 386.813451tokens/s] loss=5.6394 avg=7.8068 rate=0.0000200 step=43
2019-11-18T03:38:18.944346-08:00 [552 | 124.7582] Running opt_apply...
2019-11-18T03:38:21.607239-08:00 [552 | 127.4211 | 2.78s | 368.106441tokens/s] loss=6.6980 avg=7.7763 rate=0.0000200 step=44
2019-11-18T03:38:21.716341-08:00 [553 | 127.5302] Running opt_apply...
2019-11-18T03:38:24.324065-08:00 [553 | 130.1379 | 2.72s | 376.910282tokens/s] loss=5.7737 avg=7.7222 rate=0.0000200 step=45
2019-11-18T03:38:24.428214-08:00 [554 | 130.2420] Running opt_apply...
2019-11-18T03:38:27.044733-08:00 [554 | 132.8586 | 2.72s | 376.378407tokens/s] loss=7.1827 avg=7.7079 rate=0.0000200 step=46
2019-11-18T03:38:27.180380-08:00 [555 | 132.9942] Running opt_apply...
2019-11-18T03:38:29.799357-08:00 [555 | 135.6132 | 2.75s | 371.738590tokens/s] loss=7.4880 avg=7.7021 rate=0.0000200 step=47
2019-11-18T03:38:29.909644-08:00 [556 | 135.7235] Running opt_apply...
2019-11-18T03:38:32.564469-08:00 [556 | 138.3783 | 2.77s | 370.328558tokens/s] loss=5.3392 avg=7.6413 rate=0.0000200 step=48
2019-11-18T03:38:32.668184-08:00 [557 | 138.4820] Running opt_apply...
2019-11-18T03:38:35.215472-08:00 [557 | 141.0293 | 2.65s | 386.268634tokens/s] loss=6.5609 avg=7.6140 rate=0.0000200 step=49
2019-11-18T03:38:35.318358-08:00 [558 | 141.1322] Running opt_apply...
2019-11-18T03:38:38.245658-08:00 [558 | 144.0595 | 3.03s | 337.933058tokens/s] loss=7.8078 avg=7.6188 rate=0.0000200 step=50
2019-11-18T03:38:38.349102-08:00 [559 | 144.1629] Running opt_apply...
2019-11-18T03:38:40.961456-08:00 [559 | 146.7753 | 2.72s | 377.053060tokens/s] loss=7.4876 avg=7.6156 rate=0.0000200 step=51
2019-11-18T03:38:41.059788-08:00 [560 | 146.8736] Running opt_apply...
2019-11-18T03:38:43.626298-08:00 [560 | 149.4401 | 2.66s | 384.263223tokens/s] loss=7.8128 avg=7.6204 rate=0.0000200 step=52
2019-11-18T03:38:43.730561-08:00 [561 | 149.5444] Running opt_apply...
2019-11-18T03:38:46.300071-08:00 [561 | 152.1139 | 2.67s | 382.979708tokens/s] loss=5.6933 avg=7.5744 rate=0.0000200 step=53
2019-11-18T03:38:46.414920-08:00 [562 | 152.2287] Running opt_apply...
2019-11-18T03:38:48.996222-08:00 [562 | 154.8100 | 2.70s | 379.800381tokens/s] loss=5.2249 avg=7.5190 rate=0.0000200 step=54
2019-11-18T03:38:49.097117-08:00 [563 | 154.9109] Running opt_apply...
2019-11-18T03:38:51.656258-08:00 [563 | 157.4701 | 2.66s | 384.956942tokens/s] loss=6.0421 avg=7.4847 rate=0.0000200 step=55
2019-11-18T03:38:51.769476-08:00 [564 | 157.5833] Running opt_apply...
2019-11-18T03:38:54.377461-08:00 [564 | 160.1913 | 2.72s | 376.304408tokens/s] loss=7.4242 avg=7.4833 rate=0.0000200 step=56
2019-11-18T03:38:54.562632-08:00 [565 | 160.3765] Running opt_apply...
2019-11-18T03:38:57.142199-08:00 [565 | 162.9560 | 2.76s | 370.378665tokens/s] loss=7.8122 avg=7.4908 rate=0.0000200 step=57
2019-11-18T03:38:57.261548-08:00 [566 | 163.0754] Running opt_apply...
2019-11-18T03:38:59.856996-08:00 [566 | 165.6708 | 2.71s | 377.192005tokens/s] loss=5.1741 avg=7.4390 rate=0.0000200 step=58
2019-11-18T03:38:59.962451-08:00 [567 | 165.7763] Running opt_apply...
2019-11-18T03:39:02.577029-08:00 [567 | 168.3908 | 2.72s | 376.466294tokens/s] loss=5.6455 avg=7.3994 rate=0.0000200 step=59
2019-11-18T03:39:02.682280-08:00 [568 | 168.4961] Running opt_apply...
2019-11-18T03:39:05.335308-08:00 [568 | 171.1491 | 2.76s | 371.245940tokens/s] loss=7.3045 avg=7.3973 rate=0.0000200 step=60
2019-11-18T03:39:05.448412-08:00 [569 | 171.2622] Running opt_apply...
2019-11-18T03:39:08.066360-08:00 [569 | 173.8802 | 2.73s | 374.947003tokens/s] loss=7.3260 avg=7.3958 rate=0.0000200 step=61
2019-11-18T03:39:08.185475-08:00 [570 | 173.9993] Running opt_apply...
2019-11-18T03:39:10.756839-08:00 [570 | 176.5707 | 2.69s | 380.601367tokens/s] loss=7.2740 avg=7.3932 rate=0.0000200 step=62
2019-11-18T03:39:10.867774-08:00 [571 | 176.6816] Running opt_apply...
2019-11-18T03:39:13.447268-08:00 [571 | 179.2611 | 2.69s | 380.608450tokens/s] loss=7.5399 avg=7.3963 rate=0.0000200 step=63
2019-11-18T03:39:13.557002-08:00 [572 | 179.3708] Running opt_apply...
2019-11-18T03:39:16.137337-08:00 [572 | 181.9512 | 2.69s | 380.659218tokens/s] loss=5.1853 avg=7.3502 rate=0.0000200 step=64
2019-11-18T03:39:16.249461-08:00 [573 | 182.0633] Running opt_apply...
2019-11-18T03:39:18.857678-08:00 [573 | 184.6715 | 2.72s | 376.423533tokens/s] loss=6.9926 avg=7.3428 rate=0.0000200 step=65
2019-11-18T03:39:18.982698-08:00 [574 | 184.7965] Running opt_apply...
2019-11-18T03:39:21.550803-08:00 [574 | 187.3646 | 2.69s | 380.227564tokens/s] loss=5.5986 avg=7.3072 rate=0.0000200 step=66
2019-11-18T03:39:21.663215-08:00 [575 | 187.4770] Running opt_apply...
2019-11-18T03:39:24.217611-08:00 [575 | 190.0314 | 2.67s | 383.979356tokens/s] loss=5.0750 avg=7.2621 rate=0.0000200 step=67
2019-11-18T03:39:24.321205-08:00 [576 | 190.1350] Running opt_apply...
2019-11-18T03:39:26.877937-08:00 [576 | 192.6918 | 2.66s | 384.915404tokens/s] loss=5.4212 avg=7.2253 rate=0.0000200 step=68
2019-11-18T03:39:26.976909-08:00 [577 | 192.7907] Running opt_apply...
2019-11-18T03:39:29.517426-08:00 [577 | 195.3312 | 2.64s | 387.953855tokens/s] loss=5.9650 avg=7.2004 rate=0.0000200 step=69
2019-11-18T03:39:29.632113-08:00 [578 | 195.4459] Running opt_apply...
2019-11-18T03:39:32.215212-08:00 [578 | 198.0290 | 2.70s | 379.570259tokens/s] loss=5.1841 avg=7.1608 rate=0.0000200 step=70
2019-11-18T03:39:32.321157-08:00 [579 | 198.1350] Running opt_apply...
2019-11-18T03:39:34.858445-08:00 [579 | 200.6723 | 2.64s | 387.404670tokens/s] loss=5.1736 avg=7.1223 rate=0.0000200 step=71
2019-11-18T03:39:34.961652-08:00 [580 | 200.7755] Running opt_apply...
2019-11-18T03:39:37.591745-08:00 [580 | 203.4056 | 2.73s | 374.638556tokens/s] loss=5.8757 avg=7.0983 rate=0.0000200 step=72
2019-11-18T03:39:37.704184-08:00 [581 | 203.5180] Running opt_apply...
2019-11-18T03:39:40.344976-08:00 [581 | 206.1588 | 2.75s | 371.926779tokens/s] loss=7.3106 avg=7.1023 rate=0.0000200 step=73
2019-11-18T03:39:40.461604-08:00 [582 | 206.2754] Running opt_apply...
2019-11-18T03:39:43.038912-08:00 [582 | 208.8527 | 2.69s | 380.113118tokens/s] loss=7.5662 avg=7.1111 rate=0.0000200 step=74
2019-11-18T03:39:43.147951-08:00 [583 | 208.9618] Running opt_apply...
2019-11-18T03:39:45.678981-08:00 [583 | 211.4928 | 2.64s | 387.868509tokens/s] loss=8.1256 avg=7.1301 rate=0.0000200 step=75
2019-11-18T03:39:45.794580-08:00 [584 | 211.6084] Running opt_apply...
2019-11-18T03:39:48.359067-08:00 [584 | 214.1729 | 2.68s | 382.077272tokens/s] loss=7.5782 avg=7.1384 rate=0.0000200 step=76
2019-11-18T03:39:48.463579-08:00 [585 | 214.2774] Running opt_apply...
2019-11-18T03:39:50.995350-08:00 [585 | 216.8092 | 2.64s | 388.425650tokens/s] loss=4.7563 avg=7.0946 rate=0.0000200 step=77
2019-11-18T03:39:51.122940-08:00 [586 | 216.9368] Running opt_apply...
2019-11-18T03:39:53.771836-08:00 [586 | 219.5857 | 2.78s | 368.811269tokens/s] loss=5.6468 avg=7.0681 rate=0.0000200 step=78
2019-11-18T03:39:53.884150-08:00 [587 | 219.6980] Running opt_apply...
2019-11-18T03:39:56.537779-08:00 [587 | 222.3516 | 2.77s | 370.217726tokens/s] loss=5.5387 avg=7.0405 rate=0.0000200 step=79
2019-11-18T03:39:56.643443-08:00 [588 | 222.4573] Running opt_apply...
2019-11-18T03:39:59.188738-08:00 [588 | 225.0026 | 2.65s | 386.275026tokens/s] loss=7.5918 avg=7.0504 rate=0.0000200 step=80
2019-11-18T03:39:59.302763-08:00 [589 | 225.1166] Running opt_apply...
2019-11-18T03:40:01.863133-08:00 [589 | 227.6770 | 2.67s | 382.890597tokens/s] loss=6.4401 avg=7.0395 rate=0.0000200 step=81
2019-11-18T03:40:01.987820-08:00 [590 | 227.8016] Running opt_apply...
2019-11-18T03:40:04.569225-08:00 [590 | 230.3830 | 2.71s | 378.405048tokens/s] loss=4.4123 avg=6.9930 rate=0.0000200 step=82
2019-11-18T03:40:04.674991-08:00 [591 | 230.4888] Running opt_apply...
^C^Cinterrupted