2019-04-09 09:44:58 WARN  TaskSetManager:66 - Lost task 0.0 in stage 4.0 (TID 64, 12.0.0.6, executor 0): java.lang.RuntimeException: Training failed due to exception in ParallelWrapper fit operation
	at org.deeplearning4j.spark.parameterserver.pw.SharedTrainingWrapper.run(SharedTrainingWrapper.java:552)
	at org.deeplearning4j.spark.parameterserver.functions.SharedFlatMapPathsAdapter.call(SharedFlatMapPaths.java:94)
	at org.deeplearning4j.spark.parameterserver.functions.SharedFlatMapPathsAdapter.call(SharedFlatMapPaths.java:62)
	at org.datavec.spark.transform.BaseFlatMapFunctionAdaptee.call(BaseFlatMapFunctionAdaptee.java:40)
	at org.apache.spark.api.java.JavaRDDLike$$anonfun$fn$4$1.apply(JavaRDDLike.scala:153)
	at org.apache.spark.api.java.JavaRDDLike$$anonfun$fn$4$1.apply(JavaRDDLike.scala:153)
	at org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$23.apply(RDD.scala:801)
	at org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$23.apply(RDD.scala:801)
	at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:49)
	at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324)
	at org.apache.spark.rdd.RDD.iterator(RDD.scala:288)
	at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:49)
	at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324)
	at org.apache.spark.rdd.RDD.iterator(RDD.scala:288)
	at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:49)
	at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324)
	at org.apache.spark.rdd.RDD.iterator(RDD.scala:288)
	at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:96)
	at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53)
	at org.apache.spark.scheduler.Task.run(Task.scala:109)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:345)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: org.nd4j.linalg.exception.ND4JIllegalStateException: originalLength [0] stored in encoded array doesn't match target length [24266]
	at org.deeplearning4j.optimize.solvers.accumulation.EncodedGradientsAccumulator.applyUpdate(EncodedGradientsAccumulator.java:341)
	at org.deeplearning4j.optimize.solvers.StochasticGradientDescent.optimize(StochasticGradientDescent.java:88)
	at org.deeplearning4j.optimize.Solver.optimize(Solver.java:52)
	at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.fitHelper(MultiLayerNetwork.java:1663)
	at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.fit(MultiLayerNetwork.java:1583)
	at org.deeplearning4j.spark.parameterserver.pw.SharedTrainingWrapper.run(SharedTrainingWrapper.java:464)
	... 23 more
Caused by: org.nd4j.linalg.exception.ND4JIllegalStateException: originalLength [0] stored in encoded array doesn't match target length [24266]
	at org.nd4j.linalg.jcublas.ops.executioner.CudaExecutioner.thresholdDecode(CudaExecutioner.java:2007)
	at org.deeplearning4j.optimize.solvers.accumulation.IndexedTail.smartDecompress(IndexedTail.java:339)
	at org.deeplearning4j.optimize.solvers.accumulation.IndexedTail.drainTo(IndexedTail.java:282)
	at org.deeplearning4j.optimize.solvers.accumulation.IndexedTail.drainTo(IndexedTail.java:194)
	at org.deeplearning4j.optimize.solvers.accumulation.EncodedGradientsAccumulator.applyUpdate(EncodedGradientsAccumulator.java:315)
	... 28 more