[jp@jplap bin]$ ./adam-submit transform ~/adam_work/v1/data/run2 ~/adam_work/v1/data/run3 -mark_duplicate_reads -limit_projection
Using ADAM_MAIN=org.bdgenomics.adam.cli.ADAMMain
Using SPARK_SUBMIT=/home/jp/Apps/Spark/spark-1.6.0-bin-hadoop2.6/bin/spark-submit
2016-02-09 00:19:45 WARN  NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
SLF4J: Defaulting to no-operation (NOP) logger implementation
SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
2016-02-09 00:21:43 ERROR Executor:74 - Managed memory leak detected; size = 76525545 bytes, TID = 48
2016-02-09 00:21:43 ERROR Executor:74 - Managed memory leak detected; size = 76525545 bytes, TID = 52
2016-02-09 00:21:43 ERROR Executor:95 - Exception in task 2.0 in stage 1.0 (TID 48)
java.util.NoSuchElementException: None.get
	at scala.None$.get(Option.scala:313)
	at scala.None$.get(Option.scala:311)
	at org.bdgenomics.adam.rdd.read.MarkDuplicates$.org$bdgenomics$adam$rdd$read$MarkDuplicates$$leftPositionAndLibrary$1(MarkDuplicates.scala:73)
	at org.bdgenomics.adam.rdd.read.MarkDuplicates$$anonfun$apply$3.apply(MarkDuplicates.scala:86)
	at org.bdgenomics.adam.rdd.read.MarkDuplicates$$anonfun$apply$3.apply(MarkDuplicates.scala:86)
	at org.apache.spark.rdd.RDD$$anonfun$groupBy$3$$anonfun$apply$19.apply(RDD.scala:650)
	at org.apache.spark.rdd.RDD$$anonfun$groupBy$3$$anonfun$apply$19.apply(RDD.scala:650)
	at scala.collection.Iterator$$anon$11.next(Iterator.scala:328)
	at org.apache.spark.shuffle.sort.BypassMergeSortShuffleWriter.write(BypassMergeSortShuffleWriter.java:149)
	at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:73)
	at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:41)
	at org.apache.spark.scheduler.Task.run(Task.scala:89)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)
2016-02-09 00:21:43 ERROR Executor:95 - Exception in task 6.0 in stage 1.0 (TID 52)
java.util.NoSuchElementException: None.get
	at scala.None$.get(Option.scala:313)
	at scala.None$.get(Option.scala:311)
	at org.bdgenomics.adam.rdd.read.MarkDuplicates$.org$bdgenomics$adam$rdd$read$MarkDuplicates$$leftPositionAndLibrary$1(MarkDuplicates.scala:73)
	at org.bdgenomics.adam.rdd.read.MarkDuplicates$$anonfun$apply$3.apply(MarkDuplicates.scala:86)
	at org.bdgenomics.adam.rdd.read.MarkDuplicates$$anonfun$apply$3.apply(MarkDuplicates.scala:86)
	at org.apache.spark.rdd.RDD$$anonfun$groupBy$3$$anonfun$apply$19.apply(RDD.scala:650)
	at org.apache.spark.rdd.RDD$$anonfun$groupBy$3$$anonfun$apply$19.apply(RDD.scala:650)
	at scala.collection.Iterator$$anon$11.next(Iterator.scala:328)
	at org.apache.spark.shuffle.sort.BypassMergeSortShuffleWriter.write(BypassMergeSortShuffleWriter.java:149)
	at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:73)
	at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:41)
	at org.apache.spark.scheduler.Task.run(Task.scala:89)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)
2016-02-09 00:21:44 WARN  TaskSetManager:70 - Lost task 2.0 in stage 1.0 (TID 48, localhost): java.util.NoSuchElementException: None.get
	at scala.None$.get(Option.scala:313)
	at scala.None$.get(Option.scala:311)
	at org.bdgenomics.adam.rdd.read.MarkDuplicates$.org$bdgenomics$adam$rdd$read$MarkDuplicates$$leftPositionAndLibrary$1(MarkDuplicates.scala:73)
	at org.bdgenomics.adam.rdd.read.MarkDuplicates$$anonfun$apply$3.apply(MarkDuplicates.scala:86)
	at org.bdgenomics.adam.rdd.read.MarkDuplicates$$anonfun$apply$3.apply(MarkDuplicates.scala:86)
	at org.apache.spark.rdd.RDD$$anonfun$groupBy$3$$anonfun$apply$19.apply(RDD.scala:650)
	at org.apache.spark.rdd.RDD$$anonfun$groupBy$3$$anonfun$apply$19.apply(RDD.scala:650)
	at scala.collection.Iterator$$anon$11.next(Iterator.scala:328)
	at org.apache.spark.shuffle.sort.BypassMergeSortShuffleWriter.write(BypassMergeSortShuffleWriter.java:149)
	at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:73)
	at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:41)
	at org.apache.spark.scheduler.Task.run(Task.scala:89)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)

2016-02-09 00:21:44 ERROR TaskSetManager:74 - Task 2 in stage 1.0 failed 1 times; aborting job
Command body threw exception:
org.apache.spark.SparkException: Job aborted due to stage failure: Task 2 in stage 1.0 failed 1 times, most recent failure: Lost task 2.0 in stage 1.0 (TID 48, localhost): java.util.NoSuchElementException: None.get
	at scala.None$.get(Option.scala:313)
	at scala.None$.get(Option.scala:311)
	at org.bdgenomics.adam.rdd.read.MarkDuplicates$.org$bdgenomics$adam$rdd$read$MarkDuplicates$$leftPositionAndLibrary$1(MarkDuplicates.scala:73)
	at org.bdgenomics.adam.rdd.read.MarkDuplicates$$anonfun$apply$3.apply(MarkDuplicates.scala:86)
	at org.bdgenomics.adam.rdd.read.MarkDuplicates$$anonfun$apply$3.apply(MarkDuplicates.scala:86)
	at org.apache.spark.rdd.RDD$$anonfun$groupBy$3$$anonfun$apply$19.apply(RDD.scala:650)
	at org.apache.spark.rdd.RDD$$anonfun$groupBy$3$$anonfun$apply$19.apply(RDD.scala:650)
	at scala.collection.Iterator$$anon$11.next(Iterator.scala:328)
	at org.apache.spark.shuffle.sort.BypassMergeSortShuffleWriter.write(BypassMergeSortShuffleWriter.java:149)
	at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:73)
	at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:41)
	at org.apache.spark.scheduler.Task.run(Task.scala:89)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)

Driver stacktrace:
Exception in thread "main" 2016-02-09 00:21:44 ERROR Executor:74 - Managed memory leak detected; size = 79630424 bytes, TID = 53
2016-02-09 00:21:44 ERROR Executor:74 - Managed memory leak detected; size = 5452202 bytes, TID = 50
2016-02-09 00:21:44 ERROR Executor:74 - Managed memory leak detected; size = 37525956 bytes, TID = 49
org.apache.spark.SparkException: Job aborted due to stage failure: Task 2 in stage 1.0 failed 1 times, most recent failure: Lost task 2.0 in stage 1.0 (TID 48, localhost): java.util.NoSuchElementException: None.get
	at scala.None$.get(Option.scala:313)
	at scala.None$.get(Option.scala:311)
	at org.bdgenomics.adam.rdd.read.MarkDuplicates$.org$bdgenomics$adam$rdd$read$MarkDuplicates$$leftPositionAndLibrary$1(MarkDuplicates.scala:73)
	at org.bdgenomics.adam.rdd.read.MarkDuplicates$$anonfun$apply$3.apply(MarkDuplicates.scala:86)
	at org.bdgenomics.adam.rdd.read.MarkDuplicates$$anonfun$apply$3.apply(MarkDuplicates.scala:86)
	at org.apache.spark.rdd.RDD$$anonfun$groupBy$3$$anonfun$apply$19.apply(RDD.scala:650)
	at org.apache.spark.rdd.RDD$$anonfun$groupBy$3$$anonfun$apply$19.apply(RDD.scala:650)
	at scala.collection.Iterator$$anon$11.next(Iterator.scala:328)
	at org.apache.spark.shuffle.sort.BypassMergeSortShuffleWriter.write(BypassMergeSortShuffleWriter.java:149)
	at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:73)
	at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:41)
	at org.apache.spark.scheduler.Task.run(Task.scala:89)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)

Driver stacktrace:
	at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1431)
	at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1419)
	at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1418)
	at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
	at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47)
	at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1418)
	at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:799)
	at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:799)
	at scala.Option.foreach(Option.scala:236)
	at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:799)
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1640)
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1599)
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1588)
	at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)
	at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:620)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:1832)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:1845)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:1922)
	at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsNewAPIHadoopDataset$1.apply$mcV$sp(PairRDDFunctions.scala:1146)
	at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsNewAPIHadoopDataset$1.apply(PairRDDFunctions.scala:1074)
	at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsNewAPIHadoopDataset$1.apply(PairRDDFunctions.scala:1074)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:150)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:111)
	at org.apache.spark.rdd.RDD.withScope(RDD.scala:316)
	at org.apache.spark.rdd.PairRDDFunctions.saveAsNewAPIHadoopDataset(PairRDDFunctions.scala:1074)
	at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsNewAPIHadoopFile$2.apply$mcV$sp(PairRDDFunctions.scala:994)
	at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsNewAPIHadoopFile$2.apply(PairRDDFunctions.scala:985)
	at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsNewAPIHadoopFile$2.apply(PairRDDFunctions.scala:985)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:150)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:111)
	at org.apache.spark.rdd.RDD.withScope(RDD.scala:316)
	at org.apache.spark.rdd.PairRDDFunctions.saveAsNewAPIHadoopFile(PairRDDFunctions.scala:985)
	at org.apache.spark.rdd.InstrumentedPairRDDFunctions.saveAsNewAPIHadoopFile(InstrumentedPairRDDFunctions.scala:487)
	at org.bdgenomics.adam.rdd.ADAMRDDFunctions$$anonfun$adamParquetSave$1.apply$mcV$sp(ADAMRDDFunctions.scala:77)
	at org.bdgenomics.adam.rdd.ADAMRDDFunctions$$anonfun$adamParquetSave$1.apply(ADAMRDDFunctions.scala:61)
	at org.bdgenomics.adam.rdd.ADAMRDDFunctions$$anonfun$adamParquetSave$1.apply(ADAMRDDFunctions.scala:61)
	at org.apache.spark.rdd.Timer.time(Timer.scala:57)
	at org.bdgenomics.adam.rdd.ADAMRDDFunctions.adamParquetSave(ADAMRDDFunctions.scala:61)
	at org.bdgenomics.adam.rdd.ADAMRDDFunctions.adamParquetSave(ADAMRDDFunctions.scala:46)
	at org.bdgenomics.adam.rdd.read.AlignmentRecordRDDFunctions.saveAsParquet(AlignmentRecordRDDFunctions.scala:199)
	at org.bdgenomics.adam.rdd.read.AlignmentRecordRDDFunctions.adamSave(AlignmentRecordRDDFunctions.scala:251)
	at org.bdgenomics.adam.cli.Transform.run(Transform.scala:315)
	at org.bdgenomics.utils.cli.BDGSparkCommand$class.run(BDGCommand.scala:54)
	at org.bdgenomics.adam.cli.Transform.run(Transform.scala:119)
	at org.bdgenomics.adam.cli.ADAMMain.apply(ADAMMain.scala:136)
	at org.bdgenomics.adam.cli.ADAMMain$.main(ADAMMain.scala:76)
	at org.bdgenomics.adam.cli.ADAMMain.main(ADAMMain.scala)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:497)
	at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:731)
	at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:181)
	at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:206)
	at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:121)
	at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
Caused by: java.util.NoSuchElementException: None.get
	at scala.None$.get(Option.scala:313)
	at scala.None$.get(Option.scala:311)
	at org.bdgenomics.adam.rdd.read.MarkDuplicates$.org$bdgenomics$adam$rdd$read$MarkDuplicates$$leftPositionAndLibrary$1(MarkDuplicates.scala:73)
	at org.bdgenomics.adam.rdd.read.MarkDuplicates$$anonfun$apply$3.apply(MarkDuplicates.scala:86)
	at org.bdgenomics.adam.rdd.read.MarkDuplicates$$anonfun$apply$3.apply(MarkDuplicates.scala:86)
	at org.apache.spark.rdd.RDD$$anonfun$groupBy$3$$anonfun$apply$19.apply(RDD.scala:650)
	at org.apache.spark.rdd.RDD$$anonfun$groupBy$3$$anonfun$apply$19.apply(RDD.scala:650)
	at scala.collection.Iterator$$anon$11.next(Iterator.scala:328)
	at org.apache.spark.shuffle.sort.BypassMergeSortShuffleWriter.write(BypassMergeSortShuffleWriter.java:149)
	at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:73)
	at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:41)
	at org.apache.spark.scheduler.Task.run(Task.scala:89)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)
2016-02-09 00:21:44 WARN  TaskSetManager:70 - Lost task 1.0 in stage 1.0 (TID 47, localhost): TaskKilled (killed intentionally)
2016-02-09 00:21:44 WARN  TaskSetManager:70 - Lost task 4.0 in stage 1.0 (TID 50, localhost): TaskKilled (killed intentionally)
2016-02-09 00:21:44 WARN  TaskSetManager:70 - Lost task 8.0 in stage 1.0 (TID 54, localhost): TaskKilled (killed intentionally)
2016-02-09 00:21:44 WARN  TaskSetManager:70 - Lost task 7.0 in stage 1.0 (TID 53, localhost): TaskKilled (killed intentionally)
2016-02-09 00:21:44 WARN  TaskSetManager:70 - Lost task 0.0 in stage 1.0 (TID 46, localhost): TaskKilled (killed intentionally)
Feb 9, 2016 12:19:47 AM INFO: org.apache.parquet.hadoop.ParquetInputFormat: Total input paths to process : 46
2016-02-09 00:21:44 WARN  TaskSetManager:70 - Lost task 3.0 in stage 1.0 (TID 49, localhost): TaskKilled (killed intentionally)
2016-02-09 00:21:44 ERROR Executor:74 - Managed memory leak detected; size = 81182856 bytes, TID = 51
2016-02-09 00:21:44 ERROR Executor:95 - Exception in task 5.0 in stage 1.0 (TID 51)
java.util.NoSuchElementException: None.get
	at scala.None$.get(Option.scala:313)
	at scala.None$.get(Option.scala:311)
	at org.bdgenomics.adam.rdd.read.MarkDuplicates$.org$bdgenomics$adam$rdd$read$MarkDuplicates$$leftPositionAndLibrary$1(MarkDuplicates.scala:73)
	at org.bdgenomics.adam.rdd.read.MarkDuplicates$$anonfun$apply$3.apply(MarkDuplicates.scala:86)
	at org.bdgenomics.adam.rdd.read.MarkDuplicates$$anonfun$apply$3.apply(MarkDuplicates.scala:86)
	at org.apache.spark.rdd.RDD$$anonfun$groupBy$3$$anonfun$apply$19.apply(RDD.scala:650)
	at org.apache.spark.rdd.RDD$$anonfun$groupBy$3$$anonfun$apply$19.apply(RDD.scala:650)
	at scala.collection.Iterator$$anon$11.next(Iterator.scala:328)
	at org.apache.spark.shuffle.sort.BypassMergeSortShuffleWriter.write(BypassMergeSortShuffleWriter.java:149)
	at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:73)
	at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:41)
	at org.apache.spark.scheduler.Task.run(Task.scala:89)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)