error processing batch job Traceback (most recent call last): File "batch_job.py", line 328, in main run_driver() File "batch_job.py", line 301, in run_driver run_job( File "/opt/venv/lib64/python3.8/site-packages/openeogeotrellis/utils.py", line 43, in memory_logging_wrapper return function(*args, **kwargs) File "batch_job.py", line 360, in run_job result = ProcessGraphDeserializer.evaluate(process_graph, env=env, do_dry_run=tracer) File "/opt/venv/lib64/python3.8/site-packages/openeo_driver/ProcessGraphDeserializer.py", line 320, in evaluate return convert_node(result_node, env=env) File "/opt/venv/lib64/python3.8/site-packages/openeo_driver/ProcessGraphDeserializer.py", line 326, in convert_node return apply_process( File "/opt/venv/lib64/python3.8/site-packages/openeo_driver/ProcessGraphDeserializer.py", line 1380, in apply_process args = {name: convert_node(expr, env=env) for (name, expr) in sorted(args.items())} File "/opt/venv/lib64/python3.8/site-packages/openeo_driver/ProcessGraphDeserializer.py", line 1380, in <dictcomp> args = {name: convert_node(expr, env=env) for (name, expr) in sorted(args.items())} File "/opt/venv/lib64/python3.8/site-packages/openeo_driver/ProcessGraphDeserializer.py", line 331, in convert_node return convert_node(processGraph['node'], env=env) File "/opt/venv/lib64/python3.8/site-packages/openeo_driver/ProcessGraphDeserializer.py", line 326, in convert_node return apply_process( File "/opt/venv/lib64/python3.8/site-packages/openeo_driver/ProcessGraphDeserializer.py", line 1475, in apply_process return process_function(args=args, env=env) File "/opt/venv/lib64/python3.8/site-packages/openeo_driver/ProcessGraphDeserializer.py", line 1050, in aggregate_spatial return cube.aggregate_spatial(geometries=geoms, reducer=reduce_pg, target_dimension=target_dimension) File "/opt/venv/lib64/python3.8/site-packages/openeogeotrellis/geopysparkdatacube.py", line 1254, in aggregate_spatial return self.zonal_statistics(geometries,single_process) File "/opt/venv/lib64/python3.8/site-packages/openeogeotrellis/geopysparkdatacube.py", line 1332, in zonal_statistics self._compute_stats_geotrellis().compute_average_timeseries_from_datacube( File "/opt/spark3_2_0/python/lib/py4j-0.10.9.2-src.zip/py4j/java_gateway.py", line 1309, in __call__ return_value = get_return_value( File "/opt/spark3_2_0/python/lib/py4j-0.10.9.2-src.zip/py4j/protocol.py", line 326, in get_return_value raise Py4JJavaError( py4j.protocol.Py4JJavaError: An error occurred while calling o957.compute_average_timeseries_from_datacube. : org.apache.spark.SparkException: Job aborted due to stage failure: Task 163 in stage 13.0 failed 4 times, most recent failure: Lost task 163.3 in stage 13.0 (TID 2111) (epod031.vgt.vito.be executor 2): org.openeo.geotrellissentinelhub.SentinelHubException: Sentinel Hub returned an error response: HTTP/1.1 500 Internal Server Error with body: {"error":{"status":500,"reason":"Internal Server Error","message":"Illegal request to s3://usgs-landsat/collection02/level-2/standard/tm/1986/196/023/LT05_L2SP_196023_19860226_20200917_02_T1/LT05_L2SP_196023_19860226_20200917_02_T1_SR_B4.TIF. HTTP Status: '403'","code":"RENDERER_EXCEPTION"}} request: POST https://services-uswest2.sentinel-hub.com/api/v1/process with body: { "input": { "bounds": { "bbox": [488820.0, 5831640.0, 496500.0, 5839320.0], "properties": { "crs": "http://www.opengis.net/def/crs/EPSG/0/32632" } }, "data": [ { "type": "landsat-tm-l2", "dataFilter": {"timeRange":{"from":"1986-02-26T00:00:00Z","to":"1986-02-27T00:00:00Z"}}, "processing": {} } ] }, "output": { "width": 256, "height": 256, "responses": [ { "identifier": "default", "format": { "type": "image/tiff" } } ] }, "evalscript": "//VERSION=3\nfunction setup() {\n return {\n input: [{\n \"bands\": [\"B04\", \"B05\"]\n }],\n output: {\n bands: 2,\n sampleType: \"FLOAT32\",\n }\n };\n}\n\nfunction evaluatePixel(sample) {\n return [sample.B04, sample.B05];\n}" } at org.openeo.geotrellissentinelhub.SentinelHubException$.apply(SentinelHubException.scala:19) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$8(ProcessApi.scala:130) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$8$adapted(ProcessApi.scala:120) at scalaj.http.HttpRequest.$anonfun$toResponse$17(Http.scala:422) at scala.Option.getOrElse(Option.scala:189) at scalaj.http.HttpRequest.$anonfun$toResponse$14(Http.scala:414) at scala.Option.getOrElse(Option.scala:189) at scalaj.http.HttpRequest.toResponse(Http.scala:414) at scalaj.http.HttpRequest.doConnection(Http.scala:368) at scalaj.http.HttpRequest.exec(Http.scala:343) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$7(ProcessApi.scala:120) at org.openeo.geotrellissentinelhub.package$$anon$1.get(package.scala:60) at net.jodah.failsafe.Functions.lambda$get$0(Functions.java:46) at net.jodah.failsafe.RetryPolicyExecutor.lambda$supply$0(RetryPolicyExecutor.java:65) at net.jodah.failsafe.Execution.executeSync(Execution.java:128) at net.jodah.failsafe.FailsafeExecutor.call(FailsafeExecutor.java:378) at net.jodah.failsafe.FailsafeExecutor.get(FailsafeExecutor.java:68) at org.openeo.geotrellissentinelhub.package$.withRetries(package.scala:59) at org.openeo.geotrellissentinelhub.DefaultProcessApi.getTile(ProcessApi.scala:119) at org.openeo.geotrellissentinelhub.PyramidFactory.$anonfun$datacube_seq$1(PyramidFactory.scala:193) at org.openeo.geotrellissentinelhub.MemoizedRlGuardAdapterCachedAccessTokenWithAuthApiFallbackAuthorizer.authorized(Authorizer.scala:46) at org.openeo.geotrellissentinelhub.PyramidFactory.authorized(PyramidFactory.scala:56) at org.openeo.geotrellissentinelhub.PyramidFactory.org$openeo$geotrellissentinelhub$PyramidFactory$$getTile$1(PyramidFactory.scala:191) at org.openeo.geotrellissentinelhub.PyramidFactory.org$openeo$geotrellissentinelhub$PyramidFactory$$dataTile$1(PyramidFactory.scala:201) at org.openeo.geotrellissentinelhub.PyramidFactory.loadMasked$1(PyramidFactory.scala:226) at org.openeo.geotrellissentinelhub.PyramidFactory.$anonfun$datacube_seq$17(PyramidFactory.scala:286) at scala.collection.Iterator$$anon$10.next(Iterator.scala:459) at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:512) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:511) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:223) at org.apache.spark.storage.memory.MemoryStore.putIteratorAsBytes(MemoryStore.scala:352) at org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1498) at org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1408) at org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1472) at org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1295) at org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:384) at org.apache.spark.rdd.RDD.iterator(RDD.scala:335) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52) at org.apache.spark.scheduler.Task.run(Task.scala:131) at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:506) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1462) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:509) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) Driver stacktrace: at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2403) at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2352) at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2351) at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62) at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49) at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2351) at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1109) at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1109) at scala.Option.foreach(Option.scala:407) at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1109) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2591) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2533) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2522) at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49) at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:898) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2214) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2235) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2254) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2279) at org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1030) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112) at org.apache.spark.rdd.RDD.withScope(RDD.scala:414) at org.apache.spark.rdd.RDD.collect(RDD.scala:1029) at org.openeo.geotrellis.aggregate_polygon.AggregatePolygonProcess.$anonfun$computeAverageTimeSeries$3(AggregatePolygonProcess.scala:70) at scala.collection.parallel.mutable.ParArray$ParArrayIterator.flatmap2combiner(ParArray.scala:419) at scala.collection.parallel.ParIterableLike$FlatMap.leaf(ParIterableLike.scala:1074) at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67) at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56) at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50) at scala.collection.parallel.ParIterableLike$FlatMap.tryLeaf(ParIterableLike.scala:1070) at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.internal(Tasks.scala:160) at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.internal$(Tasks.scala:157) at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.internal(Tasks.scala:440) at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute(Tasks.scala:150) at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute$(Tasks.scala:149) at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:440) at java.base/java.util.concurrent.RecursiveAction.exec(RecursiveAction.java:189) at java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290) at java.base/java.util.concurrent.ForkJoinTask.doJoin(ForkJoinTask.java:396) at java.base/java.util.concurrent.ForkJoinTask.join(ForkJoinTask.java:721) at scala.collection.parallel.ForkJoinTasks$WrappedTask.sync(Tasks.scala:379) at scala.collection.parallel.ForkJoinTasks$WrappedTask.sync$(Tasks.scala:379) at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.sync(Tasks.scala:440) at scala.collection.parallel.ForkJoinTasks.executeAndWaitResult(Tasks.scala:423) at scala.collection.parallel.ForkJoinTasks.executeAndWaitResult$(Tasks.scala:416) at scala.collection.parallel.ForkJoinTaskSupport.executeAndWaitResult(TaskSupport.scala:60) at scala.collection.parallel.ExecutionContextTasks.executeAndWaitResult(Tasks.scala:555) at scala.collection.parallel.ExecutionContextTasks.executeAndWaitResult$(Tasks.scala:555) at scala.collection.parallel.ExecutionContextTaskSupport.executeAndWaitResult(TaskSupport.scala:84) at scala.collection.parallel.ParIterableLike$ResultMapping.leaf(ParIterableLike.scala:960) at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67) at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56) at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50) at scala.collection.parallel.ParIterableLike$ResultMapping.tryLeaf(ParIterableLike.scala:955) at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute(Tasks.scala:153) at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute$(Tasks.scala:149) at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:440) at java.base/java.util.concurrent.RecursiveAction.exec(RecursiveAction.java:189) at java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290) at java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020) at java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656) at java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594) at java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183) Suppressed: org.apache.spark.SparkException: Job aborted due to stage failure: Task 163 in stage 17.0 failed 4 times, most recent failure: Lost task 163.3 in stage 17.0 (TID 2485) (epod033.vgt.vito.be executor 3): org.openeo.geotrellissentinelhub.SentinelHubException: Sentinel Hub returned an error response: HTTP/1.1 500 Internal Server Error with body: {"error":{"status":500,"reason":"Internal Server Error","message":"Illegal request to s3://usgs-landsat/collection02/level-2/standard/tm/1986/196/023/LT05_L2SP_196023_19860226_20200917_02_T1/LT05_L2SP_196023_19860226_20200917_02_T1_SR_B4.TIF. HTTP Status: '403'","code":"RENDERER_EXCEPTION"}} request: POST https://services-uswest2.sentinel-hub.com/api/v1/process with body: { "input": { "bounds": { "bbox": [488820.0, 5831640.0, 496500.0, 5839320.0], "properties": { "crs": "http://www.opengis.net/def/crs/EPSG/0/32632" } }, "data": [ { "type": "landsat-tm-l2", "dataFilter": {"timeRange":{"from":"1986-02-26T00:00:00Z","to":"1986-02-27T00:00:00Z"}}, "processing": {} } ] }, "output": { "width": 256, "height": 256, "responses": [ { "identifier": "default", "format": { "type": "image/tiff" } } ] }, "evalscript": "//VERSION=3\nfunction setup() {\n return {\n input: [{\n \"bands\": [\"B04\", \"B05\"]\n }],\n output: {\n bands: 2,\n sampleType: \"FLOAT32\",\n }\n };\n}\n\nfunction evaluatePixel(sample) {\n return [sample.B04, sample.B05];\n}" } at org.openeo.geotrellissentinelhub.SentinelHubException$.apply(SentinelHubException.scala:19) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$8(ProcessApi.scala:130) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$8$adapted(ProcessApi.scala:120) at scalaj.http.HttpRequest.$anonfun$toResponse$17(Http.scala:422) at scala.Option.getOrElse(Option.scala:189) at scalaj.http.HttpRequest.$anonfun$toResponse$14(Http.scala:414) at scala.Option.getOrElse(Option.scala:189) at scalaj.http.HttpRequest.toResponse(Http.scala:414) at scalaj.http.HttpRequest.doConnection(Http.scala:368) at scalaj.http.HttpRequest.exec(Http.scala:343) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$7(ProcessApi.scala:120) at org.openeo.geotrellissentinelhub.package$$anon$1.get(package.scala:60) at net.jodah.failsafe.Functions.lambda$get$0(Functions.java:46) at net.jodah.failsafe.RetryPolicyExecutor.lambda$supply$0(RetryPolicyExecutor.java:65) at net.jodah.failsafe.Execution.executeSync(Execution.java:128) at net.jodah.failsafe.FailsafeExecutor.call(FailsafeExecutor.java:378) at net.jodah.failsafe.FailsafeExecutor.get(FailsafeExecutor.java:68) at org.openeo.geotrellissentinelhub.package$.withRetries(package.scala:59) at org.openeo.geotrellissentinelhub.DefaultProcessApi.getTile(ProcessApi.scala:119) at org.openeo.geotrellissentinelhub.PyramidFactory.$anonfun$datacube_seq$1(PyramidFactory.scala:193) at org.openeo.geotrellissentinelhub.MemoizedRlGuardAdapterCachedAccessTokenWithAuthApiFallbackAuthorizer.authorized(Authorizer.scala:46) at org.openeo.geotrellissentinelhub.PyramidFactory.authorized(PyramidFactory.scala:56) at org.openeo.geotrellissentinelhub.PyramidFactory.org$openeo$geotrellissentinelhub$PyramidFactory$$getTile$1(PyramidFactory.scala:191) at org.openeo.geotrellissentinelhub.PyramidFactory.org$openeo$geotrellissentinelhub$PyramidFactory$$dataTile$1(PyramidFactory.scala:201) at org.openeo.geotrellissentinelhub.PyramidFactory.loadMasked$1(PyramidFactory.scala:226) at org.openeo.geotrellissentinelhub.PyramidFactory.$anonfun$datacube_seq$17(PyramidFactory.scala:286) at scala.collection.Iterator$$anon$10.next(Iterator.scala:459) at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:512) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:511) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:223) at org.apache.spark.storage.memory.MemoryStore.putIteratorAsBytes(MemoryStore.scala:352) at org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1498) at org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1408) at org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1472) at org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1295) at org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:384) at org.apache.spark.rdd.RDD.iterator(RDD.scala:335) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52) at org.apache.spark.scheduler.Task.run(Task.scala:131) at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:506) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1462) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:509) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) Driver stacktrace: at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2403) at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2352) at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2351) at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62) at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49) at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2351) at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1109) at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1109) at scala.Option.foreach(Option.scala:407) at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1109) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2591) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2533) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2522) at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49) at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:898) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2214) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2235) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2254) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2279) at org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1030) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112) at org.apache.spark.rdd.RDD.withScope(RDD.scala:414) at org.apache.spark.rdd.RDD.collect(RDD.scala:1029) at org.openeo.geotrellis.aggregate_polygon.AggregatePolygonProcess.$anonfun$computeAverageTimeSeries$3(AggregatePolygonProcess.scala:70) at scala.collection.parallel.mutable.ParArray$ParArrayIterator.flatmap2combiner(ParArray.scala:419) at scala.collection.parallel.ParIterableLike$FlatMap.leaf(ParIterableLike.scala:1074) at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67) at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56) at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50) at scala.collection.parallel.ParIterableLike$FlatMap.tryLeaf(ParIterableLike.scala:1070) at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.internal(Tasks.scala:170) ... 34 more Caused by: org.openeo.geotrellissentinelhub.SentinelHubException: Sentinel Hub returned an error response: HTTP/1.1 500 Internal Server Error with body: {"error":{"status":500,"reason":"Internal Server Error","message":"Illegal request to s3://usgs-landsat/collection02/level-2/standard/tm/1986/196/023/LT05_L2SP_196023_19860226_20200917_02_T1/LT05_L2SP_196023_19860226_20200917_02_T1_SR_B4.TIF. HTTP Status: '403'","code":"RENDERER_EXCEPTION"}} request: POST https://services-uswest2.sentinel-hub.com/api/v1/process with body: { "input": { "bounds": { "bbox": [488820.0, 5831640.0, 496500.0, 5839320.0], "properties": { "crs": "http://www.opengis.net/def/crs/EPSG/0/32632" } }, "data": [ { "type": "landsat-tm-l2", "dataFilter": {"timeRange":{"from":"1986-02-26T00:00:00Z","to":"1986-02-27T00:00:00Z"}}, "processing": {} } ] }, "output": { "width": 256, "height": 256, "responses": [ { "identifier": "default", "format": { "type": "image/tiff" } } ] }, "evalscript": "//VERSION=3\nfunction setup() {\n return {\n input: [{\n \"bands\": [\"B04\", \"B05\"]\n }],\n output: {\n bands: 2,\n sampleType: \"FLOAT32\",\n }\n };\n}\n\nfunction evaluatePixel(sample) {\n return [sample.B04, sample.B05];\n}" } at org.openeo.geotrellissentinelhub.SentinelHubException$.apply(SentinelHubException.scala:19) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$8(ProcessApi.scala:130) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$8$adapted(ProcessApi.scala:120) at scalaj.http.HttpRequest.$anonfun$toResponse$17(Http.scala:422) at scala.Option.getOrElse(Option.scala:189) at scalaj.http.HttpRequest.$anonfun$toResponse$14(Http.scala:414) at scala.Option.getOrElse(Option.scala:189) at scalaj.http.HttpRequest.toResponse(Http.scala:414) at scalaj.http.HttpRequest.doConnection(Http.scala:368) at scalaj.http.HttpRequest.exec(Http.scala:343) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$7(ProcessApi.scala:120) at org.openeo.geotrellissentinelhub.package$$anon$1.get(package.scala:60) at net.jodah.failsafe.Functions.lambda$get$0(Functions.java:46) at net.jodah.failsafe.RetryPolicyExecutor.lambda$supply$0(RetryPolicyExecutor.java:65) at net.jodah.failsafe.Execution.executeSync(Execution.java:128) at net.jodah.failsafe.FailsafeExecutor.call(FailsafeExecutor.java:378) at net.jodah.failsafe.FailsafeExecutor.get(FailsafeExecutor.java:68) at org.openeo.geotrellissentinelhub.package$.withRetries(package.scala:59) at org.openeo.geotrellissentinelhub.DefaultProcessApi.getTile(ProcessApi.scala:119) at org.openeo.geotrellissentinelhub.PyramidFactory.$anonfun$datacube_seq$1(PyramidFactory.scala:193) at org.openeo.geotrellissentinelhub.MemoizedRlGuardAdapterCachedAccessTokenWithAuthApiFallbackAuthorizer.authorized(Authorizer.scala:46) at org.openeo.geotrellissentinelhub.PyramidFactory.authorized(PyramidFactory.scala:56) at org.openeo.geotrellissentinelhub.PyramidFactory.org$openeo$geotrellissentinelhub$PyramidFactory$$getTile$1(PyramidFactory.scala:191) at org.openeo.geotrellissentinelhub.PyramidFactory.org$openeo$geotrellissentinelhub$PyramidFactory$$dataTile$1(PyramidFactory.scala:201) at org.openeo.geotrellissentinelhub.PyramidFactory.loadMasked$1(PyramidFactory.scala:226) at org.openeo.geotrellissentinelhub.PyramidFactory.$anonfun$datacube_seq$17(PyramidFactory.scala:286) at scala.collection.Iterator$$anon$10.next(Iterator.scala:459) at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:512) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:511) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:223) at org.apache.spark.storage.memory.MemoryStore.putIteratorAsBytes(MemoryStore.scala:352) at org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1498) at org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1408) at org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1472) at org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1295) at org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:384) at org.apache.spark.rdd.RDD.iterator(RDD.scala:335) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52) at org.apache.spark.scheduler.Task.run(Task.scala:131) at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:506) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1462) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:509) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) Suppressed: org.apache.spark.SparkException: Job aborted due to stage failure: Task 163 in stage 21.0 failed 4 times, most recent failure: Lost task 163.3 in stage 21.0 (TID 3162) (epod067.vgt.vito.be executor 14): org.openeo.geotrellissentinelhub.SentinelHubException: Sentinel Hub returned an error response: HTTP/1.1 500 Internal Server Error with body: {"error":{"status":500,"reason":"Internal Server Error","message":"Illegal request to s3://usgs-landsat/collection02/level-2/standard/tm/1986/196/023/LT05_L2SP_196023_19860226_20200917_02_T1/LT05_L2SP_196023_19860226_20200917_02_T1_SR_B4.TIF. HTTP Status: '403'","code":"RENDERER_EXCEPTION"}} request: POST https://services-uswest2.sentinel-hub.com/api/v1/process with body: { "input": { "bounds": { "bbox": [488820.0, 5831640.0, 496500.0, 5839320.0], "properties": { "crs": "http://www.opengis.net/def/crs/EPSG/0/32632" } }, "data": [ { "type": "landsat-tm-l2", "dataFilter": {"timeRange":{"from":"1986-02-26T00:00:00Z","to":"1986-02-27T00:00:00Z"}}, "processing": {} } ] }, "output": { "width": 256, "height": 256, "responses": [ { "identifier": "default", "format": { "type": "image/tiff" } } ] }, "evalscript": "//VERSION=3\nfunction setup() {\n return {\n input: [{\n \"bands\": [\"B04\", \"B05\"]\n }],\n output: {\n bands: 2,\n sampleType: \"FLOAT32\",\n }\n };\n}\n\nfunction evaluatePixel(sample) {\n return [sample.B04, sample.B05];\n}" } at org.openeo.geotrellissentinelhub.SentinelHubException$.apply(SentinelHubException.scala:19) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$8(ProcessApi.scala:130) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$8$adapted(ProcessApi.scala:120) at scalaj.http.HttpRequest.$anonfun$toResponse$17(Http.scala:422) at scala.Option.getOrElse(Option.scala:189) at scalaj.http.HttpRequest.$anonfun$toResponse$14(Http.scala:414) at scala.Option.getOrElse(Option.scala:189) at scalaj.http.HttpRequest.toResponse(Http.scala:414) at scalaj.http.HttpRequest.doConnection(Http.scala:368) at scalaj.http.HttpRequest.exec(Http.scala:343) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$7(ProcessApi.scala:120) at org.openeo.geotrellissentinelhub.package$$anon$1.get(package.scala:60) at net.jodah.failsafe.Functions.lambda$get$0(Functions.java:46) at net.jodah.failsafe.RetryPolicyExecutor.lambda$supply$0(RetryPolicyExecutor.java:65) at net.jodah.failsafe.Execution.executeSync(Execution.java:128) at net.jodah.failsafe.FailsafeExecutor.call(FailsafeExecutor.java:378) at net.jodah.failsafe.FailsafeExecutor.get(FailsafeExecutor.java:68) at org.openeo.geotrellissentinelhub.package$.withRetries(package.scala:59) at org.openeo.geotrellissentinelhub.DefaultProcessApi.getTile(ProcessApi.scala:119) at org.openeo.geotrellissentinelhub.PyramidFactory.$anonfun$datacube_seq$1(PyramidFactory.scala:193) at org.openeo.geotrellissentinelhub.MemoizedRlGuardAdapterCachedAccessTokenWithAuthApiFallbackAuthorizer.authorized(Authorizer.scala:46) at org.openeo.geotrellissentinelhub.PyramidFactory.authorized(PyramidFactory.scala:56) at org.openeo.geotrellissentinelhub.PyramidFactory.org$openeo$geotrellissentinelhub$PyramidFactory$$getTile$1(PyramidFactory.scala:191) at org.openeo.geotrellissentinelhub.PyramidFactory.org$openeo$geotrellissentinelhub$PyramidFactory$$dataTile$1(PyramidFactory.scala:201) at org.openeo.geotrellissentinelhub.PyramidFactory.loadMasked$1(PyramidFactory.scala:226) at org.openeo.geotrellissentinelhub.PyramidFactory.$anonfun$datacube_seq$17(PyramidFactory.scala:286) at scala.collection.Iterator$$anon$10.next(Iterator.scala:459) at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:512) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:511) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:223) at org.apache.spark.storage.memory.MemoryStore.putIteratorAsBytes(MemoryStore.scala:352) at org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1498) at org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1408) at org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1472) at org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1295) at org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:384) at org.apache.spark.rdd.RDD.iterator(RDD.scala:335) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52) at org.apache.spark.scheduler.Task.run(Task.scala:131) at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:506) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1462) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:509) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) Driver stacktrace: at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2403) at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2352) at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2351) at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62) at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49) at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2351) at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1109) at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1109) at scala.Option.foreach(Option.scala:407) at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1109) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2591) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2533) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2522) at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49) at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:898) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2214) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2235) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2254) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2279) at org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1030) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112) at org.apache.spark.rdd.RDD.withScope(RDD.scala:414) at org.apache.spark.rdd.RDD.collect(RDD.scala:1029) at org.openeo.geotrellis.aggregate_polygon.AggregatePolygonProcess.$anonfun$computeAverageTimeSeries$3(AggregatePolygonProcess.scala:70) at scala.collection.parallel.mutable.ParArray$ParArrayIterator.flatmap2combiner(ParArray.scala:419) at scala.collection.parallel.ParIterableLike$FlatMap.leaf(ParIterableLike.scala:1074) at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67) at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56) at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50) at scala.collection.parallel.ParIterableLike$FlatMap.tryLeaf(ParIterableLike.scala:1070) at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.internal(Tasks.scala:170) ... 34 more Caused by: org.openeo.geotrellissentinelhub.SentinelHubException: Sentinel Hub returned an error response: HTTP/1.1 500 Internal Server Error with body: {"error":{"status":500,"reason":"Internal Server Error","message":"Illegal request to s3://usgs-landsat/collection02/level-2/standard/tm/1986/196/023/LT05_L2SP_196023_19860226_20200917_02_T1/LT05_L2SP_196023_19860226_20200917_02_T1_SR_B4.TIF. HTTP Status: '403'","code":"RENDERER_EXCEPTION"}} request: POST https://services-uswest2.sentinel-hub.com/api/v1/process with body: { "input": { "bounds": { "bbox": [488820.0, 5831640.0, 496500.0, 5839320.0], "properties": { "crs": "http://www.opengis.net/def/crs/EPSG/0/32632" } }, "data": [ { "type": "landsat-tm-l2", "dataFilter": {"timeRange":{"from":"1986-02-26T00:00:00Z","to":"1986-02-27T00:00:00Z"}}, "processing": {} } ] }, "output": { "width": 256, "height": 256, "responses": [ { "identifier": "default", "format": { "type": "image/tiff" } } ] }, "evalscript": "//VERSION=3\nfunction setup() {\n return {\n input: [{\n \"bands\": [\"B04\", \"B05\"]\n }],\n output: {\n bands: 2,\n sampleType: \"FLOAT32\",\n }\n };\n}\n\nfunction evaluatePixel(sample) {\n return [sample.B04, sample.B05];\n}" } at org.openeo.geotrellissentinelhub.SentinelHubException$.apply(SentinelHubException.scala:19) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$8(ProcessApi.scala:130) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$8$adapted(ProcessApi.scala:120) at scalaj.http.HttpRequest.$anonfun$toResponse$17(Http.scala:422) at scala.Option.getOrElse(Option.scala:189) at scalaj.http.HttpRequest.$anonfun$toResponse$14(Http.scala:414) at scala.Option.getOrElse(Option.scala:189) at scalaj.http.HttpRequest.toResponse(Http.scala:414) at scalaj.http.HttpRequest.doConnection(Http.scala:368) at scalaj.http.HttpRequest.exec(Http.scala:343) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$7(ProcessApi.scala:120) at org.openeo.geotrellissentinelhub.package$$anon$1.get(package.scala:60) at net.jodah.failsafe.Functions.lambda$get$0(Functions.java:46) at net.jodah.failsafe.RetryPolicyExecutor.lambda$supply$0(RetryPolicyExecutor.java:65) at net.jodah.failsafe.Execution.executeSync(Execution.java:128) at net.jodah.failsafe.FailsafeExecutor.call(FailsafeExecutor.java:378) at net.jodah.failsafe.FailsafeExecutor.get(FailsafeExecutor.java:68) at org.openeo.geotrellissentinelhub.package$.withRetries(package.scala:59) at org.openeo.geotrellissentinelhub.DefaultProcessApi.getTile(ProcessApi.scala:119) at org.openeo.geotrellissentinelhub.PyramidFactory.$anonfun$datacube_seq$1(PyramidFactory.scala:193) at org.openeo.geotrellissentinelhub.MemoizedRlGuardAdapterCachedAccessTokenWithAuthApiFallbackAuthorizer.authorized(Authorizer.scala:46) at org.openeo.geotrellissentinelhub.PyramidFactory.authorized(PyramidFactory.scala:56) at org.openeo.geotrellissentinelhub.PyramidFactory.org$openeo$geotrellissentinelhub$PyramidFactory$$getTile$1(PyramidFactory.scala:191) at org.openeo.geotrellissentinelhub.PyramidFactory.org$openeo$geotrellissentinelhub$PyramidFactory$$dataTile$1(PyramidFactory.scala:201) at org.openeo.geotrellissentinelhub.PyramidFactory.loadMasked$1(PyramidFactory.scala:226) at org.openeo.geotrellissentinelhub.PyramidFactory.$anonfun$datacube_seq$17(PyramidFactory.scala:286) at scala.collection.Iterator$$anon$10.next(Iterator.scala:459) at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:512) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:511) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:223) at org.apache.spark.storage.memory.MemoryStore.putIteratorAsBytes(MemoryStore.scala:352) at org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1498) at org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1408) at org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1472) at org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1295) at org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:384) at org.apache.spark.rdd.RDD.iterator(RDD.scala:335) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52) at org.apache.spark.scheduler.Task.run(Task.scala:131) at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:506) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1462) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:509) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) Suppressed: org.apache.spark.SparkException: Job aborted due to stage failure: Task 164 in stage 25.0 failed 4 times, most recent failure: Lost task 164.3 in stage 25.0 (TID 3906) (epod048.vgt.vito.be executor 12): org.openeo.geotrellissentinelhub.SentinelHubException: Sentinel Hub returned an error response: HTTP/1.1 500 Internal Server Error with body: {"error":{"status":500,"reason":"Internal Server Error","message":"Illegal request to s3://usgs-landsat/collection02/level-2/standard/tm/1986/196/023/LT05_L2SP_196023_19860226_20200917_02_T1/LT05_L2SP_196023_19860226_20200917_02_T1_SR_B4.TIF. HTTP Status: '403'","code":"RENDERER_EXCEPTION"}} request: POST https://services-uswest2.sentinel-hub.com/api/v1/process with body: { "input": { "bounds": { "bbox": [496500.0, 5831640.0, 504180.0, 5839320.0], "properties": { "crs": "http://www.opengis.net/def/crs/EPSG/0/32632" } }, "data": [ { "type": "landsat-tm-l2", "dataFilter": {"timeRange":{"from":"1986-02-26T00:00:00Z","to":"1986-02-27T00:00:00Z"}}, "processing": {} } ] }, "output": { "width": 256, "height": 256, "responses": [ { "identifier": "default", "format": { "type": "image/tiff" } } ] }, "evalscript": "//VERSION=3\nfunction setup() {\n return {\n input: [{\n \"bands\": [\"B04\", \"B05\"]\n }],\n output: {\n bands: 2,\n sampleType: \"FLOAT32\",\n }\n };\n}\n\nfunction evaluatePixel(sample) {\n return [sample.B04, sample.B05];\n}" } at org.openeo.geotrellissentinelhub.SentinelHubException$.apply(SentinelHubException.scala:19) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$8(ProcessApi.scala:130) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$8$adapted(ProcessApi.scala:120) at scalaj.http.HttpRequest.$anonfun$toResponse$17(Http.scala:422) at scala.Option.getOrElse(Option.scala:189) at scalaj.http.HttpRequest.$anonfun$toResponse$14(Http.scala:414) at scala.Option.getOrElse(Option.scala:189) at scalaj.http.HttpRequest.toResponse(Http.scala:414) at scalaj.http.HttpRequest.doConnection(Http.scala:368) at scalaj.http.HttpRequest.exec(Http.scala:343) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$7(ProcessApi.scala:120) at org.openeo.geotrellissentinelhub.package$$anon$1.get(package.scala:60) at net.jodah.failsafe.Functions.lambda$get$0(Functions.java:46) at net.jodah.failsafe.RetryPolicyExecutor.lambda$supply$0(RetryPolicyExecutor.java:65) at net.jodah.failsafe.Execution.executeSync(Execution.java:128) at net.jodah.failsafe.FailsafeExecutor.call(FailsafeExecutor.java:378) at net.jodah.failsafe.FailsafeExecutor.get(FailsafeExecutor.java:68) at org.openeo.geotrellissentinelhub.package$.withRetries(package.scala:59) at org.openeo.geotrellissentinelhub.DefaultProcessApi.getTile(ProcessApi.scala:119) at org.openeo.geotrellissentinelhub.PyramidFactory.$anonfun$datacube_seq$1(PyramidFactory.scala:193) at org.openeo.geotrellissentinelhub.MemoizedRlGuardAdapterCachedAccessTokenWithAuthApiFallbackAuthorizer.authorized(Authorizer.scala:46) at org.openeo.geotrellissentinelhub.PyramidFactory.authorized(PyramidFactory.scala:56) at org.openeo.geotrellissentinelhub.PyramidFactory.org$openeo$geotrellissentinelhub$PyramidFactory$$getTile$1(PyramidFactory.scala:191) at org.openeo.geotrellissentinelhub.PyramidFactory.org$openeo$geotrellissentinelhub$PyramidFactory$$dataTile$1(PyramidFactory.scala:201) at org.openeo.geotrellissentinelhub.PyramidFactory.loadMasked$1(PyramidFactory.scala:226) at org.openeo.geotrellissentinelhub.PyramidFactory.$anonfun$datacube_seq$17(PyramidFactory.scala:286) at scala.collection.Iterator$$anon$10.next(Iterator.scala:459) at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:512) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:511) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:223) at org.apache.spark.storage.memory.MemoryStore.putIteratorAsBytes(MemoryStore.scala:352) at org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1498) at org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1408) at org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1472) at org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1295) at org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:384) at org.apache.spark.rdd.RDD.iterator(RDD.scala:335) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52) at org.apache.spark.scheduler.Task.run(Task.scala:131) at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:506) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1462) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:509) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) Driver stacktrace: at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2403) at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2352) at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2351) at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62) at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49) at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2351) at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1109) at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1109) at scala.Option.foreach(Option.scala:407) at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1109) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2591) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2533) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2522) at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49) at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:898) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2214) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2235) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2254) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2279) at org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1030) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112) at org.apache.spark.rdd.RDD.withScope(RDD.scala:414) at org.apache.spark.rdd.RDD.collect(RDD.scala:1029) at org.openeo.geotrellis.aggregate_polygon.AggregatePolygonProcess.$anonfun$computeAverageTimeSeries$3(AggregatePolygonProcess.scala:70) at scala.collection.parallel.mutable.ParArray$ParArrayIterator.flatmap2combiner(ParArray.scala:419) at scala.collection.parallel.ParIterableLike$FlatMap.leaf(ParIterableLike.scala:1074) at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67) at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56) at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50) at scala.collection.parallel.ParIterableLike$FlatMap.tryLeaf(ParIterableLike.scala:1070) at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.internal(Tasks.scala:170) ... 34 more Caused by: org.openeo.geotrellissentinelhub.SentinelHubException: Sentinel Hub returned an error response: HTTP/1.1 500 Internal Server Error with body: {"error":{"status":500,"reason":"Internal Server Error","message":"Illegal request to s3://usgs-landsat/collection02/level-2/standard/tm/1986/196/023/LT05_L2SP_196023_19860226_20200917_02_T1/LT05_L2SP_196023_19860226_20200917_02_T1_SR_B4.TIF. HTTP Status: '403'","code":"RENDERER_EXCEPTION"}} request: POST https://services-uswest2.sentinel-hub.com/api/v1/process with body: { "input": { "bounds": { "bbox": [496500.0, 5831640.0, 504180.0, 5839320.0], "properties": { "crs": "http://www.opengis.net/def/crs/EPSG/0/32632" } }, "data": [ { "type": "landsat-tm-l2", "dataFilter": {"timeRange":{"from":"1986-02-26T00:00:00Z","to":"1986-02-27T00:00:00Z"}}, "processing": {} } ] }, "output": { "width": 256, "height": 256, "responses": [ { "identifier": "default", "format": { "type": "image/tiff" } } ] }, "evalscript": "//VERSION=3\nfunction setup() {\n return {\n input: [{\n \"bands\": [\"B04\", \"B05\"]\n }],\n output: {\n bands: 2,\n sampleType: \"FLOAT32\",\n }\n };\n}\n\nfunction evaluatePixel(sample) {\n return [sample.B04, sample.B05];\n}" } at org.openeo.geotrellissentinelhub.SentinelHubException$.apply(SentinelHubException.scala:19) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$8(ProcessApi.scala:130) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$8$adapted(ProcessApi.scala:120) at scalaj.http.HttpRequest.$anonfun$toResponse$17(Http.scala:422) at scala.Option.getOrElse(Option.scala:189) at scalaj.http.HttpRequest.$anonfun$toResponse$14(Http.scala:414) at scala.Option.getOrElse(Option.scala:189) at scalaj.http.HttpRequest.toResponse(Http.scala:414) at scalaj.http.HttpRequest.doConnection(Http.scala:368) at scalaj.http.HttpRequest.exec(Http.scala:343) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$7(ProcessApi.scala:120) at org.openeo.geotrellissentinelhub.package$$anon$1.get(package.scala:60) at net.jodah.failsafe.Functions.lambda$get$0(Functions.java:46) at net.jodah.failsafe.RetryPolicyExecutor.lambda$supply$0(RetryPolicyExecutor.java:65) at net.jodah.failsafe.Execution.executeSync(Execution.java:128) at net.jodah.failsafe.FailsafeExecutor.call(FailsafeExecutor.java:378) at net.jodah.failsafe.FailsafeExecutor.get(FailsafeExecutor.java:68) at org.openeo.geotrellissentinelhub.package$.withRetries(package.scala:59) at org.openeo.geotrellissentinelhub.DefaultProcessApi.getTile(ProcessApi.scala:119) at org.openeo.geotrellissentinelhub.PyramidFactory.$anonfun$datacube_seq$1(PyramidFactory.scala:193) at org.openeo.geotrellissentinelhub.MemoizedRlGuardAdapterCachedAccessTokenWithAuthApiFallbackAuthorizer.authorized(Authorizer.scala:46) at org.openeo.geotrellissentinelhub.PyramidFactory.authorized(PyramidFactory.scala:56) at org.openeo.geotrellissentinelhub.PyramidFactory.org$openeo$geotrellissentinelhub$PyramidFactory$$getTile$1(PyramidFactory.scala:191) at org.openeo.geotrellissentinelhub.PyramidFactory.org$openeo$geotrellissentinelhub$PyramidFactory$$dataTile$1(PyramidFactory.scala:201) at org.openeo.geotrellissentinelhub.PyramidFactory.loadMasked$1(PyramidFactory.scala:226) at org.openeo.geotrellissentinelhub.PyramidFactory.$anonfun$datacube_seq$17(PyramidFactory.scala:286) at scala.collection.Iterator$$anon$10.next(Iterator.scala:459) at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:512) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:511) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:223) at org.apache.spark.storage.memory.MemoryStore.putIteratorAsBytes(MemoryStore.scala:352) at org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1498) at org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1408) at org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1472) at org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1295) at org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:384) at org.apache.spark.rdd.RDD.iterator(RDD.scala:335) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52) at org.apache.spark.scheduler.Task.run(Task.scala:131) at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:506) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1462) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:509) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) Suppressed: org.apache.spark.SparkException: Job aborted due to stage failure: Task 163 in stage 15.0 failed 4 times, most recent failure: Lost task 163.3 in stage 15.0 (TID 2195) (epod104.vgt.vito.be executor 5): org.openeo.geotrellissentinelhub.SentinelHubException: Sentinel Hub returned an error response: HTTP/1.1 500 Internal Server Error with body: {"error":{"status":500,"reason":"Internal Server Error","message":"Illegal request to s3://usgs-landsat/collection02/level-2/standard/tm/1986/196/023/LT05_L2SP_196023_19860226_20200917_02_T1/LT05_L2SP_196023_19860226_20200917_02_T1_SR_B4.TIF. HTTP Status: '403'","code":"RENDERER_EXCEPTION"}} request: POST https://services-uswest2.sentinel-hub.com/api/v1/process with body: { "input": { "bounds": { "bbox": [488820.0, 5831640.0, 496500.0, 5839320.0], "properties": { "crs": "http://www.opengis.net/def/crs/EPSG/0/32632" } }, "data": [ { "type": "landsat-tm-l2", "dataFilter": {"timeRange":{"from":"1986-02-26T00:00:00Z","to":"1986-02-27T00:00:00Z"}}, "processing": {} } ] }, "output": { "width": 256, "height": 256, "responses": [ { "identifier": "default", "format": { "type": "image/tiff" } } ] }, "evalscript": "//VERSION=3\nfunction setup() {\n return {\n input: [{\n \"bands\": [\"B04\", \"B05\"]\n }],\n output: {\n bands: 2,\n sampleType: \"FLOAT32\",\n }\n };\n}\n\nfunction evaluatePixel(sample) {\n return [sample.B04, sample.B05];\n}" } at org.openeo.geotrellissentinelhub.SentinelHubException$.apply(SentinelHubException.scala:19) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$8(ProcessApi.scala:130) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$8$adapted(ProcessApi.scala:120) at scalaj.http.HttpRequest.$anonfun$toResponse$17(Http.scala:422) at scala.Option.getOrElse(Option.scala:189) at scalaj.http.HttpRequest.$anonfun$toResponse$14(Http.scala:414) at scala.Option.getOrElse(Option.scala:189) at scalaj.http.HttpRequest.toResponse(Http.scala:414) at scalaj.http.HttpRequest.doConnection(Http.scala:368) at scalaj.http.HttpRequest.exec(Http.scala:343) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$7(ProcessApi.scala:120) at org.openeo.geotrellissentinelhub.package$$anon$1.get(package.scala:60) at net.jodah.failsafe.Functions.lambda$get$0(Functions.java:46) at net.jodah.failsafe.RetryPolicyExecutor.lambda$supply$0(RetryPolicyExecutor.java:65) at net.jodah.failsafe.Execution.executeSync(Execution.java:128) at net.jodah.failsafe.FailsafeExecutor.call(FailsafeExecutor.java:378) at net.jodah.failsafe.FailsafeExecutor.get(FailsafeExecutor.java:68) at org.openeo.geotrellissentinelhub.package$.withRetries(package.scala:59) at org.openeo.geotrellissentinelhub.DefaultProcessApi.getTile(ProcessApi.scala:119) at org.openeo.geotrellissentinelhub.PyramidFactory.$anonfun$datacube_seq$1(PyramidFactory.scala:193) at org.openeo.geotrellissentinelhub.MemoizedRlGuardAdapterCachedAccessTokenWithAuthApiFallbackAuthorizer.authorized(Authorizer.scala:46) at org.openeo.geotrellissentinelhub.PyramidFactory.authorized(PyramidFactory.scala:56) at org.openeo.geotrellissentinelhub.PyramidFactory.org$openeo$geotrellissentinelhub$PyramidFactory$$getTile$1(PyramidFactory.scala:191) at org.openeo.geotrellissentinelhub.PyramidFactory.org$openeo$geotrellissentinelhub$PyramidFactory$$dataTile$1(PyramidFactory.scala:201) at org.openeo.geotrellissentinelhub.PyramidFactory.loadMasked$1(PyramidFactory.scala:226) at org.openeo.geotrellissentinelhub.PyramidFactory.$anonfun$datacube_seq$17(PyramidFactory.scala:286) at scala.collection.Iterator$$anon$10.next(Iterator.scala:459) at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:512) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:511) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:223) at org.apache.spark.storage.memory.MemoryStore.putIteratorAsBytes(MemoryStore.scala:352) at org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1498) at org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1408) at org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1472) at org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1295) at org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:384) at org.apache.spark.rdd.RDD.iterator(RDD.scala:335) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52) at org.apache.spark.scheduler.Task.run(Task.scala:131) at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:506) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1462) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:509) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) Driver stacktrace: at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2403) at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2352) at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2351) at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62) at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49) at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2351) at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1109) at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1109) at scala.Option.foreach(Option.scala:407) at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1109) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2591) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2533) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2522) at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49) at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:898) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2214) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2235) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2254) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2279) at org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1030) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112) at org.apache.spark.rdd.RDD.withScope(RDD.scala:414) at org.apache.spark.rdd.RDD.collect(RDD.scala:1029) at org.openeo.geotrellis.aggregate_polygon.AggregatePolygonProcess.$anonfun$computeAverageTimeSeries$3(AggregatePolygonProcess.scala:70) at scala.collection.parallel.mutable.ParArray$ParArrayIterator.flatmap2combiner(ParArray.scala:419) at scala.collection.parallel.ParIterableLike$FlatMap.leaf(ParIterableLike.scala:1074) at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67) at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56) at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50) at scala.collection.parallel.ParIterableLike$FlatMap.tryLeaf(ParIterableLike.scala:1070) at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.internal(Tasks.scala:160) at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.internal$(Tasks.scala:157) at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.internal(Tasks.scala:440) at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute(Tasks.scala:150) ... 8 more Suppressed: org.apache.spark.SparkException: Job aborted due to stage failure: Task 163 in stage 19.0 failed 4 times, most recent failure: Lost task 163.3 in stage 19.0 (TID 2998) (epod111.vgt.vito.be executor 9): org.openeo.geotrellissentinelhub.SentinelHubException: Sentinel Hub returned an error response: HTTP/1.1 500 Internal Server Error with body: {"error":{"status":500,"reason":"Internal Server Error","message":"Illegal request to s3://usgs-landsat/collection02/level-2/standard/tm/1986/196/023/LT05_L2SP_196023_19860226_20200917_02_T1/LT05_L2SP_196023_19860226_20200917_02_T1_SR_B5.TIF. HTTP Status: '403'","code":"RENDERER_EXCEPTION"}} request: POST https://services-uswest2.sentinel-hub.com/api/v1/process with body: { "input": { "bounds": { "bbox": [488820.0, 5831640.0, 496500.0, 5839320.0], "properties": { "crs": "http://www.opengis.net/def/crs/EPSG/0/32632" } }, "data": [ { "type": "landsat-tm-l2", "dataFilter": {"timeRange":{"from":"1986-02-26T00:00:00Z","to":"1986-02-27T00:00:00Z"}}, "processing": {} } ] }, "output": { "width": 256, "height": 256, "responses": [ { "identifier": "default", "format": { "type": "image/tiff" } } ] }, "evalscript": "//VERSION=3\nfunction setup() {\n return {\n input: [{\n \"bands\": [\"B04\", \"B05\"]\n }],\n output: {\n bands: 2,\n sampleType: \"FLOAT32\",\n }\n };\n}\n\nfunction evaluatePixel(sample) {\n return [sample.B04, sample.B05];\n}" } at org.openeo.geotrellissentinelhub.SentinelHubException$.apply(SentinelHubException.scala:19) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$8(ProcessApi.scala:130) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$8$adapted(ProcessApi.scala:120) at scalaj.http.HttpRequest.$anonfun$toResponse$17(Http.scala:422) at scala.Option.getOrElse(Option.scala:189) at scalaj.http.HttpRequest.$anonfun$toResponse$14(Http.scala:414) at scala.Option.getOrElse(Option.scala:189) at scalaj.http.HttpRequest.toResponse(Http.scala:414) at scalaj.http.HttpRequest.doConnection(Http.scala:368) at scalaj.http.HttpRequest.exec(Http.scala:343) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$7(ProcessApi.scala:120) at org.openeo.geotrellissentinelhub.package$$anon$1.get(package.scala:60) at net.jodah.failsafe.Functions.lambda$get$0(Functions.java:46) at net.jodah.failsafe.RetryPolicyExecutor.lambda$supply$0(RetryPolicyExecutor.java:65) at net.jodah.failsafe.Execution.executeSync(Execution.java:128) at net.jodah.failsafe.FailsafeExecutor.call(FailsafeExecutor.java:378) at net.jodah.failsafe.FailsafeExecutor.get(FailsafeExecutor.java:68) at org.openeo.geotrellissentinelhub.package$.withRetries(package.scala:59) at org.openeo.geotrellissentinelhub.DefaultProcessApi.getTile(ProcessApi.scala:119) at org.openeo.geotrellissentinelhub.PyramidFactory.$anonfun$datacube_seq$1(PyramidFactory.scala:193) at org.openeo.geotrellissentinelhub.MemoizedRlGuardAdapterCachedAccessTokenWithAuthApiFallbackAuthorizer.authorized(Authorizer.scala:46) at org.openeo.geotrellissentinelhub.PyramidFactory.authorized(PyramidFactory.scala:56) at org.openeo.geotrellissentinelhub.PyramidFactory.org$openeo$geotrellissentinelhub$PyramidFactory$$getTile$1(PyramidFactory.scala:191) at org.openeo.geotrellissentinelhub.PyramidFactory.org$openeo$geotrellissentinelhub$PyramidFactory$$dataTile$1(PyramidFactory.scala:201) at org.openeo.geotrellissentinelhub.PyramidFactory.loadMasked$1(PyramidFactory.scala:226) at org.openeo.geotrellissentinelhub.PyramidFactory.$anonfun$datacube_seq$17(PyramidFactory.scala:286) at scala.collection.Iterator$$anon$10.next(Iterator.scala:459) at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:512) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:511) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:223) at org.apache.spark.storage.memory.MemoryStore.putIteratorAsBytes(MemoryStore.scala:352) at org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1498) at org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1408) at org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1472) at org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1295) at org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:384) at org.apache.spark.rdd.RDD.iterator(RDD.scala:335) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52) at org.apache.spark.scheduler.Task.run(Task.scala:131) at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:506) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1462) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:509) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) Driver stacktrace: at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2403) at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2352) at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2351) at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62) at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49) at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2351) at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1109) at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1109) at scala.Option.foreach(Option.scala:407) at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1109) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2591) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2533) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2522) at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49) at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:898) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2214) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2235) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2254) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2279) at org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1030) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112) at org.apache.spark.rdd.RDD.withScope(RDD.scala:414) at org.apache.spark.rdd.RDD.collect(RDD.scala:1029) at org.openeo.geotrellis.aggregate_polygon.AggregatePolygonProcess.$anonfun$computeAverageTimeSeries$3(AggregatePolygonProcess.scala:70) at scala.collection.parallel.mutable.ParArray$ParArrayIterator.flatmap2combiner(ParArray.scala:419) at scala.collection.parallel.ParIterableLike$FlatMap.leaf(ParIterableLike.scala:1074) at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67) at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56) at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50) at scala.collection.parallel.ParIterableLike$FlatMap.tryLeaf(ParIterableLike.scala:1070) at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.internal(Tasks.scala:170) ... 11 more Caused by: org.openeo.geotrellissentinelhub.SentinelHubException: Sentinel Hub returned an error response: HTTP/1.1 500 Internal Server Error with body: {"error":{"status":500,"reason":"Internal Server Error","message":"Illegal request to s3://usgs-landsat/collection02/level-2/standard/tm/1986/196/023/LT05_L2SP_196023_19860226_20200917_02_T1/LT05_L2SP_196023_19860226_20200917_02_T1_SR_B5.TIF. HTTP Status: '403'","code":"RENDERER_EXCEPTION"}} request: POST https://services-uswest2.sentinel-hub.com/api/v1/process with body: { "input": { "bounds": { "bbox": [488820.0, 5831640.0, 496500.0, 5839320.0], "properties": { "crs": "http://www.opengis.net/def/crs/EPSG/0/32632" } }, "data": [ { "type": "landsat-tm-l2", "dataFilter": {"timeRange":{"from":"1986-02-26T00:00:00Z","to":"1986-02-27T00:00:00Z"}}, "processing": {} } ] }, "output": { "width": 256, "height": 256, "responses": [ { "identifier": "default", "format": { "type": "image/tiff" } } ] }, "evalscript": "//VERSION=3\nfunction setup() {\n return {\n input: [{\n \"bands\": [\"B04\", \"B05\"]\n }],\n output: {\n bands: 2,\n sampleType: \"FLOAT32\",\n }\n };\n}\n\nfunction evaluatePixel(sample) {\n return [sample.B04, sample.B05];\n}" } at org.openeo.geotrellissentinelhub.SentinelHubException$.apply(SentinelHubException.scala:19) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$8(ProcessApi.scala:130) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$8$adapted(ProcessApi.scala:120) at scalaj.http.HttpRequest.$anonfun$toResponse$17(Http.scala:422) at scala.Option.getOrElse(Option.scala:189) at scalaj.http.HttpRequest.$anonfun$toResponse$14(Http.scala:414) at scala.Option.getOrElse(Option.scala:189) at scalaj.http.HttpRequest.toResponse(Http.scala:414) at scalaj.http.HttpRequest.doConnection(Http.scala:368) at scalaj.http.HttpRequest.exec(Http.scala:343) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$7(ProcessApi.scala:120) at org.openeo.geotrellissentinelhub.package$$anon$1.get(package.scala:60) at net.jodah.failsafe.Functions.lambda$get$0(Functions.java:46) at net.jodah.failsafe.RetryPolicyExecutor.lambda$supply$0(RetryPolicyExecutor.java:65) at net.jodah.failsafe.Execution.executeSync(Execution.java:128) at net.jodah.failsafe.FailsafeExecutor.call(FailsafeExecutor.java:378) at net.jodah.failsafe.FailsafeExecutor.get(FailsafeExecutor.java:68) at org.openeo.geotrellissentinelhub.package$.withRetries(package.scala:59) at org.openeo.geotrellissentinelhub.DefaultProcessApi.getTile(ProcessApi.scala:119) at org.openeo.geotrellissentinelhub.PyramidFactory.$anonfun$datacube_seq$1(PyramidFactory.scala:193) at org.openeo.geotrellissentinelhub.MemoizedRlGuardAdapterCachedAccessTokenWithAuthApiFallbackAuthorizer.authorized(Authorizer.scala:46) at org.openeo.geotrellissentinelhub.PyramidFactory.authorized(PyramidFactory.scala:56) at org.openeo.geotrellissentinelhub.PyramidFactory.org$openeo$geotrellissentinelhub$PyramidFactory$$getTile$1(PyramidFactory.scala:191) at org.openeo.geotrellissentinelhub.PyramidFactory.org$openeo$geotrellissentinelhub$PyramidFactory$$dataTile$1(PyramidFactory.scala:201) at org.openeo.geotrellissentinelhub.PyramidFactory.loadMasked$1(PyramidFactory.scala:226) at org.openeo.geotrellissentinelhub.PyramidFactory.$anonfun$datacube_seq$17(PyramidFactory.scala:286) at scala.collection.Iterator$$anon$10.next(Iterator.scala:459) at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:512) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:511) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:223) at org.apache.spark.storage.memory.MemoryStore.putIteratorAsBytes(MemoryStore.scala:352) at org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1498) at org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1408) at org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1472) at org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1295) at org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:384) at org.apache.spark.rdd.RDD.iterator(RDD.scala:335) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52) at org.apache.spark.scheduler.Task.run(Task.scala:131) at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:506) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1462) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:509) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) Suppressed: org.apache.spark.SparkException: Job aborted due to stage failure: Task 247 in stage 23.0 failed 4 times, most recent failure: Lost task 247.3 in stage 23.0 (TID 3529) (epod021.vgt.vito.be executor 7): org.openeo.geotrellissentinelhub.SentinelHubException: Sentinel Hub returned an error response: HTTP/1.1 500 Internal Server Error with body: {"error":{"status":500,"reason":"Internal Server Error","message":"Illegal request to s3://usgs-landsat/collection02/level-2/standard/tm/1987/196/024/LT05_L2SP_196024_19870213_20201014_02_T2/LT05_L2SP_196024_19870213_20201014_02_T2_SR_B4.TIF. HTTP Status: '403'","code":"RENDERER_EXCEPTION"}} request: POST https://services-uswest2.sentinel-hub.com/api/v1/process with body: { "input": { "bounds": { "bbox": [488820.0, 5831640.0, 496500.0, 5839320.0], "properties": { "crs": "http://www.opengis.net/def/crs/EPSG/0/32632" } }, "data": [ { "type": "landsat-tm-l2", "dataFilter": {"timeRange":{"from":"1987-02-13T00:00:00Z","to":"1987-02-14T00:00:00Z"}}, "processing": {} } ] }, "output": { "width": 256, "height": 256, "responses": [ { "identifier": "default", "format": { "type": "image/tiff" } } ] }, "evalscript": "//VERSION=3\nfunction setup() {\n return {\n input: [{\n \"bands\": [\"B04\", \"B05\"]\n }],\n output: {\n bands: 2,\n sampleType: \"FLOAT32\",\n }\n };\n}\n\nfunction evaluatePixel(sample) {\n return [sample.B04, sample.B05];\n}" } at org.openeo.geotrellissentinelhub.SentinelHubException$.apply(SentinelHubException.scala:19) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$8(ProcessApi.scala:130) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$8$adapted(ProcessApi.scala:120) at scalaj.http.HttpRequest.$anonfun$toResponse$17(Http.scala:422) at scala.Option.getOrElse(Option.scala:189) at scalaj.http.HttpRequest.$anonfun$toResponse$14(Http.scala:414) at scala.Option.getOrElse(Option.scala:189) at scalaj.http.HttpRequest.toResponse(Http.scala:414) at scalaj.http.HttpRequest.doConnection(Http.scala:368) at scalaj.http.HttpRequest.exec(Http.scala:343) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$7(ProcessApi.scala:120) at org.openeo.geotrellissentinelhub.package$$anon$1.get(package.scala:60) at net.jodah.failsafe.Functions.lambda$get$0(Functions.java:46) at net.jodah.failsafe.RetryPolicyExecutor.lambda$supply$0(RetryPolicyExecutor.java:65) at net.jodah.failsafe.Execution.executeSync(Execution.java:128) at net.jodah.failsafe.FailsafeExecutor.call(FailsafeExecutor.java:378) at net.jodah.failsafe.FailsafeExecutor.get(FailsafeExecutor.java:68) at org.openeo.geotrellissentinelhub.package$.withRetries(package.scala:59) at org.openeo.geotrellissentinelhub.DefaultProcessApi.getTile(ProcessApi.scala:119) at org.openeo.geotrellissentinelhub.PyramidFactory.$anonfun$datacube_seq$1(PyramidFactory.scala:193) at org.openeo.geotrellissentinelhub.MemoizedRlGuardAdapterCachedAccessTokenWithAuthApiFallbackAuthorizer.authorized(Authorizer.scala:46) at org.openeo.geotrellissentinelhub.PyramidFactory.authorized(PyramidFactory.scala:56) at org.openeo.geotrellissentinelhub.PyramidFactory.org$openeo$geotrellissentinelhub$PyramidFactory$$getTile$1(PyramidFactory.scala:191) at org.openeo.geotrellissentinelhub.PyramidFactory.org$openeo$geotrellissentinelhub$PyramidFactory$$dataTile$1(PyramidFactory.scala:201) at org.openeo.geotrellissentinelhub.PyramidFactory.loadMasked$1(PyramidFactory.scala:226) at org.openeo.geotrellissentinelhub.PyramidFactory.$anonfun$datacube_seq$17(PyramidFactory.scala:286) at scala.collection.Iterator$$anon$10.next(Iterator.scala:459) at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:512) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:511) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:223) at org.apache.spark.storage.memory.MemoryStore.putIteratorAsBytes(MemoryStore.scala:352) at org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1498) at org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1408) at org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1472) at org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1295) at org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:384) at org.apache.spark.rdd.RDD.iterator(RDD.scala:335) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52) at org.apache.spark.scheduler.Task.run(Task.scala:131) at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:506) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1462) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:509) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) Driver stacktrace: at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2403) at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2352) at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2351) at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62) at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49) at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2351) at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1109) at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1109) at scala.Option.foreach(Option.scala:407) at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1109) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2591) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2533) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2522) at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49) at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:898) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2214) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2235) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2254) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2279) at org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1030) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112) at org.apache.spark.rdd.RDD.withScope(RDD.scala:414) at org.apache.spark.rdd.RDD.collect(RDD.scala:1029) at org.openeo.geotrellis.aggregate_polygon.AggregatePolygonProcess.$anonfun$computeAverageTimeSeries$3(AggregatePolygonProcess.scala:70) at scala.collection.parallel.mutable.ParArray$ParArrayIterator.flatmap2combiner(ParArray.scala:419) at scala.collection.parallel.ParIterableLike$FlatMap.leaf(ParIterableLike.scala:1074) at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67) at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56) at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50) at scala.collection.parallel.ParIterableLike$FlatMap.tryLeaf(ParIterableLike.scala:1070) at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.internal(Tasks.scala:170) ... 11 more Caused by: org.openeo.geotrellissentinelhub.SentinelHubException: Sentinel Hub returned an error response: HTTP/1.1 500 Internal Server Error with body: {"error":{"status":500,"reason":"Internal Server Error","message":"Illegal request to s3://usgs-landsat/collection02/level-2/standard/tm/1987/196/024/LT05_L2SP_196024_19870213_20201014_02_T2/LT05_L2SP_196024_19870213_20201014_02_T2_SR_B4.TIF. HTTP Status: '403'","code":"RENDERER_EXCEPTION"}} request: POST https://services-uswest2.sentinel-hub.com/api/v1/process with body: { "input": { "bounds": { "bbox": [488820.0, 5831640.0, 496500.0, 5839320.0], "properties": { "crs": "http://www.opengis.net/def/crs/EPSG/0/32632" } }, "data": [ { "type": "landsat-tm-l2", "dataFilter": {"timeRange":{"from":"1987-02-13T00:00:00Z","to":"1987-02-14T00:00:00Z"}}, "processing": {} } ] }, "output": { "width": 256, "height": 256, "responses": [ { "identifier": "default", "format": { "type": "image/tiff" } } ] }, "evalscript": "//VERSION=3\nfunction setup() {\n return {\n input: [{\n \"bands\": [\"B04\", \"B05\"]\n }],\n output: {\n bands: 2,\n sampleType: \"FLOAT32\",\n }\n };\n}\n\nfunction evaluatePixel(sample) {\n return [sample.B04, sample.B05];\n}" } at org.openeo.geotrellissentinelhub.SentinelHubException$.apply(SentinelHubException.scala:19) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$8(ProcessApi.scala:130) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$8$adapted(ProcessApi.scala:120) at scalaj.http.HttpRequest.$anonfun$toResponse$17(Http.scala:422) at scala.Option.getOrElse(Option.scala:189) at scalaj.http.HttpRequest.$anonfun$toResponse$14(Http.scala:414) at scala.Option.getOrElse(Option.scala:189) at scalaj.http.HttpRequest.toResponse(Http.scala:414) at scalaj.http.HttpRequest.doConnection(Http.scala:368) at scalaj.http.HttpRequest.exec(Http.scala:343) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$7(ProcessApi.scala:120) at org.openeo.geotrellissentinelhub.package$$anon$1.get(package.scala:60) at net.jodah.failsafe.Functions.lambda$get$0(Functions.java:46) at net.jodah.failsafe.RetryPolicyExecutor.lambda$supply$0(RetryPolicyExecutor.java:65) at net.jodah.failsafe.Execution.executeSync(Execution.java:128) at net.jodah.failsafe.FailsafeExecutor.call(FailsafeExecutor.java:378) at net.jodah.failsafe.FailsafeExecutor.get(FailsafeExecutor.java:68) at org.openeo.geotrellissentinelhub.package$.withRetries(package.scala:59) at org.openeo.geotrellissentinelhub.DefaultProcessApi.getTile(ProcessApi.scala:119) at org.openeo.geotrellissentinelhub.PyramidFactory.$anonfun$datacube_seq$1(PyramidFactory.scala:193) at org.openeo.geotrellissentinelhub.MemoizedRlGuardAdapterCachedAccessTokenWithAuthApiFallbackAuthorizer.authorized(Authorizer.scala:46) at org.openeo.geotrellissentinelhub.PyramidFactory.authorized(PyramidFactory.scala:56) at org.openeo.geotrellissentinelhub.PyramidFactory.org$openeo$geotrellissentinelhub$PyramidFactory$$getTile$1(PyramidFactory.scala:191) at org.openeo.geotrellissentinelhub.PyramidFactory.org$openeo$geotrellissentinelhub$PyramidFactory$$dataTile$1(PyramidFactory.scala:201) at org.openeo.geotrellissentinelhub.PyramidFactory.loadMasked$1(PyramidFactory.scala:226) at org.openeo.geotrellissentinelhub.PyramidFactory.$anonfun$datacube_seq$17(PyramidFactory.scala:286) at scala.collection.Iterator$$anon$10.next(Iterator.scala:459) at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:512) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:511) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:223) at org.apache.spark.storage.memory.MemoryStore.putIteratorAsBytes(MemoryStore.scala:352) at org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1498) at org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1408) at org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1472) at org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1295) at org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:384) at org.apache.spark.rdd.RDD.iterator(RDD.scala:335) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52) at org.apache.spark.scheduler.Task.run(Task.scala:131) at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:506) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1462) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:509) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) Suppressed: org.apache.spark.SparkException: Job aborted due to stage failure: Task 252 in stage 27.0 failed 4 times, most recent failure: Lost task 252.3 in stage 27.0 (TID 4261) (epod161.vgt.vito.be executor 18): org.openeo.geotrellissentinelhub.SentinelHubException: Sentinel Hub returned an error response: HTTP/1.1 500 Internal Server Error with body: {"error":{"status":500,"reason":"Internal Server Error","message":"Illegal request to s3://usgs-landsat/collection02/level-2/standard/tm/1987/197/023/LT05_L2SP_197023_19870308_20201014_02_T1/LT05_L2SP_197023_19870308_20201014_02_T1_SR_B5.TIF. HTTP Status: '403'","code":"RENDERER_EXCEPTION"}} request: POST https://services-uswest2.sentinel-hub.com/api/v1/process with body: { "input": { "bounds": { "bbox": [465780.0, 5823960.0, 473460.0, 5831640.0], "properties": { "crs": "http://www.opengis.net/def/crs/EPSG/0/32632" } }, "data": [ { "type": "landsat-tm-l2", "dataFilter": {"timeRange":{"from":"1987-03-08T00:00:00Z","to":"1987-03-09T00:00:00Z"}}, "processing": {} } ] }, "output": { "width": 256, "height": 256, "responses": [ { "identifier": "default", "format": { "type": "image/tiff" } } ] }, "evalscript": "//VERSION=3\nfunction setup() {\n return {\n input: [{\n \"bands\": [\"B04\", \"B05\"]\n }],\n output: {\n bands: 2,\n sampleType: \"FLOAT32\",\n }\n };\n}\n\nfunction evaluatePixel(sample) {\n return [sample.B04, sample.B05];\n}" } at org.openeo.geotrellissentinelhub.SentinelHubException$.apply(SentinelHubException.scala:19) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$8(ProcessApi.scala:130) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$8$adapted(ProcessApi.scala:120) at scalaj.http.HttpRequest.$anonfun$toResponse$17(Http.scala:422) at scala.Option.getOrElse(Option.scala:189) at scalaj.http.HttpRequest.$anonfun$toResponse$14(Http.scala:414) at scala.Option.getOrElse(Option.scala:189) at scalaj.http.HttpRequest.toResponse(Http.scala:414) at scalaj.http.HttpRequest.doConnection(Http.scala:368) at scalaj.http.HttpRequest.exec(Http.scala:343) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$7(ProcessApi.scala:120) at org.openeo.geotrellissentinelhub.package$$anon$1.get(package.scala:60) at net.jodah.failsafe.Functions.lambda$get$0(Functions.java:46) at net.jodah.failsafe.RetryPolicyExecutor.lambda$supply$0(RetryPolicyExecutor.java:65) at net.jodah.failsafe.Execution.executeSync(Execution.java:128) at net.jodah.failsafe.FailsafeExecutor.call(FailsafeExecutor.java:378) at net.jodah.failsafe.FailsafeExecutor.get(FailsafeExecutor.java:68) at org.openeo.geotrellissentinelhub.package$.withRetries(package.scala:59) at org.openeo.geotrellissentinelhub.DefaultProcessApi.getTile(ProcessApi.scala:119) at org.openeo.geotrellissentinelhub.PyramidFactory.$anonfun$datacube_seq$1(PyramidFactory.scala:193) at org.openeo.geotrellissentinelhub.MemoizedRlGuardAdapterCachedAccessTokenWithAuthApiFallbackAuthorizer.authorized(Authorizer.scala:46) at org.openeo.geotrellissentinelhub.PyramidFactory.authorized(PyramidFactory.scala:56) at org.openeo.geotrellissentinelhub.PyramidFactory.org$openeo$geotrellissentinelhub$PyramidFactory$$getTile$1(PyramidFactory.scala:191) at org.openeo.geotrellissentinelhub.PyramidFactory.org$openeo$geotrellissentinelhub$PyramidFactory$$dataTile$1(PyramidFactory.scala:201) at org.openeo.geotrellissentinelhub.PyramidFactory.loadMasked$1(PyramidFactory.scala:226) at org.openeo.geotrellissentinelhub.PyramidFactory.$anonfun$datacube_seq$17(PyramidFactory.scala:286) at scala.collection.Iterator$$anon$10.next(Iterator.scala:459) at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:512) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:511) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:223) at org.apache.spark.storage.memory.MemoryStore.putIteratorAsBytes(MemoryStore.scala:352) at org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1498) at org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1408) at org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1472) at org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1295) at org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:384) at org.apache.spark.rdd.RDD.iterator(RDD.scala:335) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52) at org.apache.spark.scheduler.Task.run(Task.scala:131) at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:506) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1462) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:509) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) Driver stacktrace: at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2403) at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2352) at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2351) at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62) at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49) at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2351) at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1109) at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1109) at scala.Option.foreach(Option.scala:407) at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1109) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2591) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2533) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2522) at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49) at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:898) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2214) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2235) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2254) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2279) at org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1030) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112) at org.apache.spark.rdd.RDD.withScope(RDD.scala:414) at org.apache.spark.rdd.RDD.collect(RDD.scala:1029) at org.openeo.geotrellis.aggregate_polygon.AggregatePolygonProcess.$anonfun$computeAverageTimeSeries$3(AggregatePolygonProcess.scala:70) at scala.collection.parallel.mutable.ParArray$ParArrayIterator.flatmap2combiner(ParArray.scala:419) at scala.collection.parallel.ParIterableLike$FlatMap.leaf(ParIterableLike.scala:1074) at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67) at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56) at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50) at scala.collection.parallel.ParIterableLike$FlatMap.tryLeaf(ParIterableLike.scala:1070) at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.internal(Tasks.scala:170) ... 11 more Caused by: org.openeo.geotrellissentinelhub.SentinelHubException: Sentinel Hub returned an error response: HTTP/1.1 500 Internal Server Error with body: {"error":{"status":500,"reason":"Internal Server Error","message":"Illegal request to s3://usgs-landsat/collection02/level-2/standard/tm/1987/197/023/LT05_L2SP_197023_19870308_20201014_02_T1/LT05_L2SP_197023_19870308_20201014_02_T1_SR_B5.TIF. HTTP Status: '403'","code":"RENDERER_EXCEPTION"}} request: POST https://services-uswest2.sentinel-hub.com/api/v1/process with body: { "input": { "bounds": { "bbox": [465780.0, 5823960.0, 473460.0, 5831640.0], "properties": { "crs": "http://www.opengis.net/def/crs/EPSG/0/32632" } }, "data": [ { "type": "landsat-tm-l2", "dataFilter": {"timeRange":{"from":"1987-03-08T00:00:00Z","to":"1987-03-09T00:00:00Z"}}, "processing": {} } ] }, "output": { "width": 256, "height": 256, "responses": [ { "identifier": "default", "format": { "type": "image/tiff" } } ] }, "evalscript": "//VERSION=3\nfunction setup() {\n return {\n input: [{\n \"bands\": [\"B04\", \"B05\"]\n }],\n output: {\n bands: 2,\n sampleType: \"FLOAT32\",\n }\n };\n}\n\nfunction evaluatePixel(sample) {\n return [sample.B04, sample.B05];\n}" } at org.openeo.geotrellissentinelhub.SentinelHubException$.apply(SentinelHubException.scala:19) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$8(ProcessApi.scala:130) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$8$adapted(ProcessApi.scala:120) at scalaj.http.HttpRequest.$anonfun$toResponse$17(Http.scala:422) at scala.Option.getOrElse(Option.scala:189) at scalaj.http.HttpRequest.$anonfun$toResponse$14(Http.scala:414) at scala.Option.getOrElse(Option.scala:189) at scalaj.http.HttpRequest.toResponse(Http.scala:414) at scalaj.http.HttpRequest.doConnection(Http.scala:368) at scalaj.http.HttpRequest.exec(Http.scala:343) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$7(ProcessApi.scala:120) at org.openeo.geotrellissentinelhub.package$$anon$1.get(package.scala:60) at net.jodah.failsafe.Functions.lambda$get$0(Functions.java:46) at net.jodah.failsafe.RetryPolicyExecutor.lambda$supply$0(RetryPolicyExecutor.java:65) at net.jodah.failsafe.Execution.executeSync(Execution.java:128) at net.jodah.failsafe.FailsafeExecutor.call(FailsafeExecutor.java:378) at net.jodah.failsafe.FailsafeExecutor.get(FailsafeExecutor.java:68) at org.openeo.geotrellissentinelhub.package$.withRetries(package.scala:59) at org.openeo.geotrellissentinelhub.DefaultProcessApi.getTile(ProcessApi.scala:119) at org.openeo.geotrellissentinelhub.PyramidFactory.$anonfun$datacube_seq$1(PyramidFactory.scala:193) at org.openeo.geotrellissentinelhub.MemoizedRlGuardAdapterCachedAccessTokenWithAuthApiFallbackAuthorizer.authorized(Authorizer.scala:46) at org.openeo.geotrellissentinelhub.PyramidFactory.authorized(PyramidFactory.scala:56) at org.openeo.geotrellissentinelhub.PyramidFactory.org$openeo$geotrellissentinelhub$PyramidFactory$$getTile$1(PyramidFactory.scala:191) at org.openeo.geotrellissentinelhub.PyramidFactory.org$openeo$geotrellissentinelhub$PyramidFactory$$dataTile$1(PyramidFactory.scala:201) at org.openeo.geotrellissentinelhub.PyramidFactory.loadMasked$1(PyramidFactory.scala:226) at org.openeo.geotrellissentinelhub.PyramidFactory.$anonfun$datacube_seq$17(PyramidFactory.scala:286) at scala.collection.Iterator$$anon$10.next(Iterator.scala:459) at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:512) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:511) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:223) at org.apache.spark.storage.memory.MemoryStore.putIteratorAsBytes(MemoryStore.scala:352) at org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1498) at org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1408) at org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1472) at org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1295) at org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:384) at org.apache.spark.rdd.RDD.iterator(RDD.scala:335) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52) at org.apache.spark.scheduler.Task.run(Task.scala:131) at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:506) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1462) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:509) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) Caused by: org.openeo.geotrellissentinelhub.SentinelHubException: Sentinel Hub returned an error response: HTTP/1.1 500 Internal Server Error with body: {"error":{"status":500,"reason":"Internal Server Error","message":"Illegal request to s3://usgs-landsat/collection02/level-2/standard/tm/1986/196/023/LT05_L2SP_196023_19860226_20200917_02_T1/LT05_L2SP_196023_19860226_20200917_02_T1_SR_B4.TIF. HTTP Status: '403'","code":"RENDERER_EXCEPTION"}} request: POST https://services-uswest2.sentinel-hub.com/api/v1/process with body: { "input": { "bounds": { "bbox": [488820.0, 5831640.0, 496500.0, 5839320.0], "properties": { "crs": "http://www.opengis.net/def/crs/EPSG/0/32632" } }, "data": [ { "type": "landsat-tm-l2", "dataFilter": {"timeRange":{"from":"1986-02-26T00:00:00Z","to":"1986-02-27T00:00:00Z"}}, "processing": {} } ] }, "output": { "width": 256, "height": 256, "responses": [ { "identifier": "default", "format": { "type": "image/tiff" } } ] }, "evalscript": "//VERSION=3\nfunction setup() {\n return {\n input: [{\n \"bands\": [\"B04\", \"B05\"]\n }],\n output: {\n bands: 2,\n sampleType: \"FLOAT32\",\n }\n };\n}\n\nfunction evaluatePixel(sample) {\n return [sample.B04, sample.B05];\n}" } at org.openeo.geotrellissentinelhub.SentinelHubException$.apply(SentinelHubException.scala:19) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$8(ProcessApi.scala:130) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$8$adapted(ProcessApi.scala:120) at scalaj.http.HttpRequest.$anonfun$toResponse$17(Http.scala:422) at scala.Option.getOrElse(Option.scala:189) at scalaj.http.HttpRequest.$anonfun$toResponse$14(Http.scala:414) at scala.Option.getOrElse(Option.scala:189) at scalaj.http.HttpRequest.toResponse(Http.scala:414) at scalaj.http.HttpRequest.doConnection(Http.scala:368) at scalaj.http.HttpRequest.exec(Http.scala:343) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$7(ProcessApi.scala:120) at org.openeo.geotrellissentinelhub.package$$anon$1.get(package.scala:60) at net.jodah.failsafe.Functions.lambda$get$0(Functions.java:46) at net.jodah.failsafe.RetryPolicyExecutor.lambda$supply$0(RetryPolicyExecutor.java:65) at net.jodah.failsafe.Execution.executeSync(Execution.java:128) at net.jodah.failsafe.FailsafeExecutor.call(FailsafeExecutor.java:378) at net.jodah.failsafe.FailsafeExecutor.get(FailsafeExecutor.java:68) at org.openeo.geotrellissentinelhub.package$.withRetries(package.scala:59) at org.openeo.geotrellissentinelhub.DefaultProcessApi.getTile(ProcessApi.scala:119) at org.openeo.geotrellissentinelhub.PyramidFactory.$anonfun$datacube_seq$1(PyramidFactory.scala:193) at org.openeo.geotrellissentinelhub.MemoizedRlGuardAdapterCachedAccessTokenWithAuthApiFallbackAuthorizer.authorized(Authorizer.scala:46) at org.openeo.geotrellissentinelhub.PyramidFactory.authorized(PyramidFactory.scala:56) at org.openeo.geotrellissentinelhub.PyramidFactory.org$openeo$geotrellissentinelhub$PyramidFactory$$getTile$1(PyramidFactory.scala:191) at org.openeo.geotrellissentinelhub.PyramidFactory.org$openeo$geotrellissentinelhub$PyramidFactory$$dataTile$1(PyramidFactory.scala:201) at org.openeo.geotrellissentinelhub.PyramidFactory.loadMasked$1(PyramidFactory.scala:226) at org.openeo.geotrellissentinelhub.PyramidFactory.$anonfun$datacube_seq$17(PyramidFactory.scala:286) at scala.collection.Iterator$$anon$10.next(Iterator.scala:459) at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:512) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:511) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:223) at org.apache.spark.storage.memory.MemoryStore.putIteratorAsBytes(MemoryStore.scala:352) at org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1498) at org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1408) at org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1472) at org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1295) at org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:384) at org.apache.spark.rdd.RDD.iterator(RDD.scala:335) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52) at org.apache.spark.scheduler.Task.run(Task.scala:131) at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:506) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1462) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:509) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) Caused by: org.openeo.geotrellissentinelhub.SentinelHubException: Sentinel Hub returned an error response: HTTP/1.1 500 Internal Server Error with body: {"error":{"status":500,"reason":"Internal Server Error","message":"Illegal request to s3://usgs-landsat/collection02/level-2/standard/tm/1986/196/023/LT05_L2SP_196023_19860226_20200917_02_T1/LT05_L2SP_196023_19860226_20200917_02_T1_SR_B4.TIF. HTTP Status: '403'","code":"RENDERER_EXCEPTION"}} request: POST https://services-uswest2.sentinel-hub.com/api/v1/process with body: { "input": { "bounds": { "bbox": [488820.0, 5831640.0, 496500.0, 5839320.0], "properties": { "crs": "http://www.opengis.net/def/crs/EPSG/0/32632" } }, "data": [ { "type": "landsat-tm-l2", "dataFilter": {"timeRange":{"from":"1986-02-26T00:00:00Z","to":"1986-02-27T00:00:00Z"}}, "processing": {} } ] }, "output": { "width": 256, "height": 256, "responses": [ { "identifier": "default", "format": { "type": "image/tiff" } } ] }, "evalscript": "//VERSION=3\nfunction setup() {\n return {\n input: [{\n \"bands\": [\"B04\", \"B05\"]\n }],\n output: {\n bands: 2,\n sampleType: \"FLOAT32\",\n }\n };\n}\n\nfunction evaluatePixel(sample) {\n return [sample.B04, sample.B05];\n}" } at org.openeo.geotrellissentinelhub.SentinelHubException$.apply(SentinelHubException.scala:19) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$8(ProcessApi.scala:130) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$8$adapted(ProcessApi.scala:120) at scalaj.http.HttpRequest.$anonfun$toResponse$17(Http.scala:422) at scala.Option.getOrElse(Option.scala:189) at scalaj.http.HttpRequest.$anonfun$toResponse$14(Http.scala:414) at scala.Option.getOrElse(Option.scala:189) at scalaj.http.HttpRequest.toResponse(Http.scala:414) at scalaj.http.HttpRequest.doConnection(Http.scala:368) at scalaj.http.HttpRequest.exec(Http.scala:343) at org.openeo.geotrellissentinelhub.DefaultProcessApi.$anonfun$getTile$7(ProcessApi.scala:120) at org.openeo.geotrellissentinelhub.package$$anon$1.get(package.scala:60) at net.jodah.failsafe.Functions.lambda$get$0(Functions.java:46) at net.jodah.failsafe.RetryPolicyExecutor.lambda$supply$0(RetryPolicyExecutor.java:65) at net.jodah.failsafe.Execution.executeSync(Execution.java:128) at net.jodah.failsafe.FailsafeExecutor.call(FailsafeExecutor.java:378) at net.jodah.failsafe.FailsafeExecutor.get(FailsafeExecutor.java:68) at org.openeo.geotrellissentinelhub.package$.withRetries(package.scala:59) at org.openeo.geotrellissentinelhub.DefaultProcessApi.getTile(ProcessApi.scala:119) at org.openeo.geotrellissentinelhub.PyramidFactory.$anonfun$datacube_seq$1(PyramidFactory.scala:193) at org.openeo.geotrellissentinelhub.MemoizedRlGuardAdapterCachedAccessTokenWithAuthApiFallbackAuthorizer.authorized(Authorizer.scala:46) at org.openeo.geotrellissentinelhub.PyramidFactory.authorized(PyramidFactory.scala:56) at org.openeo.geotrellissentinelhub.PyramidFactory.org$openeo$geotrellissentinelhub$PyramidFactory$$getTile$1(PyramidFactory.scala:191) at org.openeo.geotrellissentinelhub.PyramidFactory.org$openeo$geotrellissentinelhub$PyramidFactory$$dataTile$1(PyramidFactory.scala:201) at org.openeo.geotrellissentinelhub.PyramidFactory.loadMasked$1(PyramidFactory.scala:226) at org.openeo.geotrellissentinelhub.PyramidFactory.$anonfun$datacube_seq$17(PyramidFactory.scala:286) at scala.collection.Iterator$$anon$10.next(Iterator.scala:459) at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:512) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:511) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:223) at org.apache.spark.storage.memory.MemoryStore.putIteratorAsBytes(MemoryStore.scala:352) at org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1498) at org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1408) at org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1472) at org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1295) at org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:384) at org.apache.spark.rdd.RDD.iterator(RDD.scala:335) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52) at org.apache.spark.scheduler.Task.run(Task.scala:131) at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:506) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1462) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:509) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829)