API tools faq
paste
Advertisement
Guest User

Untitled

a guest
Jul 6th, 2018
281
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
SPARK 12.05 KB | None | 0 0
raw download clone embed print report
  1. Py4JJavaError: An error occurred while calling o44.collectToPython.
  2. : org.apache.spark.sql.catalyst.errors.package$TreeNodeException: execute, tree:
  3. Exchange rangepartitioning(latitude#30 ASC NULLS FIRST, longitude#31 ASC NULLS FIRST, 200)
  4. +- *HashAggregate(keys=[substring(field#40, 1, 6)#58, substring(field2)#59], functions=[], output=[latitude#30, longitude#31])
  5.    +- Exchange hashpartitioning(substring(field#40, 1, 6)#58, substring(field2#41, 1, 6)#59, 200)
  6.       +- *HashAggregate(keys=[substring(field#40, 1, 6) AS substring(field#40, 1, 6)#58, substring(field2#41, 1, 6) AS substring(field2#41, 1, 6)#59], functions=[], output=[substring(field#40, 1, 6)#58, substring(field2#41, 1, 6)#59])
  7.         +- HiveTableScan [field#40, field2#41], HiveTableRelation `scheme`.`table_name`, org.apache.hadoop.hive.ql.io.orc.OrcSerde, [FIELDS LIST], [isnotnull(field3#52), isnotnull(field4#51), isnotnull(field5#50), (field6#50 >= 17328), (time_key#50 <= 17365), (cast(fileld3#52 as int) = 5201), (cast(field4#51 as int) = 52)]
  8.     at org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:56)
  9.     at org.apache.spark.sql.execution.exchange.ShuffleExchange.doExecute(ShuffleExchange.scala:115)
  10.     at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:117)
  11.     at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:117)
  12.     at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:138)
  13.     at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
  14.     at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:135)
  15.     at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:116)
  16.     at org.apache.spark.sql.execution.InputAdapter.inputRDDs(WholeStageCodegenExec.scala:252)
  17.     at org.apache.spark.sql.execution.SortExec.inputRDDs(SortExec.scala:121)
  18.     at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:386)
  19.     at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:117)
  20.     at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:117)
  21.     at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:138)
  22.     at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
  23.     at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:135)
  24.     at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:116)
  25.     at org.apache.spark.sql.execution.SparkPlan.getByteArrayRdd(SparkPlan.scala:228)
  26.     at org.apache.spark.sql.execution.SparkPlan.executeCollect(SparkPlan.scala:275)
  27.     at org.apache.spark.sql.Dataset$$anonfun$collectToPython$1.apply$mcI$sp(Dataset.scala:2804)
  28.     at org.apache.spark.sql.Dataset$$anonfun$collectToPython$1.apply(Dataset.scala:2801)
  29.     at org.apache.spark.sql.Dataset$$anonfun$collectToPython$1.apply(Dataset.scala:2801)
  30.     at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:65)
  31.     at org.apache.spark.sql.Dataset.withNewExecutionId(Dataset.scala:2824)
  32.     at org.apache.spark.sql.Dataset.collectToPython(Dataset.scala:2801)
  33.     at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
  34.     at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
  35.     at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  36.     at java.lang.reflect.Method.invoke(Method.java:498)
  37.     at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
  38.     at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
  39.     at py4j.Gateway.invoke(Gateway.java:280)
  40.     at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
  41.     at py4j.commands.CallCommand.execute(CallCommand.java:79)
  42.     at py4j.GatewayConnection.run(GatewayConnection.java:214)
  43.     at java.lang.Thread.run(Thread.java:748)
  44. Caused by: org.apache.spark.sql.catalyst.errors.package$TreeNodeException: execute, tree:
  45. Exchange hashpartitioning(substring(field#40, 1, 6)#58, substring(field2#41, 1, 6)#59, 200)
  46. +- *HashAggregate(keys=[substring(field#40, 1, 6) AS substring(field#40, 1, 6)#58, substring(field2#41, 1, 6) AS substring(cell_list_avg_lon#41, 1, 6)#59], functions=[], output=[substring(field#40, 1, 6)#58, substring(field2#41, 1, 6)#59])
  47.    +- HiveTableScan [field#40, field2#41], HiveTableRelation `scheme`.`table_name`, org.apache.hadoop.hive.ql.io.orc.OrcSerde, [FIELDS LIST], [isnotnull(field3#52), isnotnull(field4#51), isnotnull(field5#50), (field6#50 >= 17328), (time_key#50 <= 17365), (cast(fileld3#52 as int) = 5201), (cast(field4#51 as int) = 52)]
  48.  
  49.     at org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:56)
  50.     at org.apache.spark.sql.execution.exchange.ShuffleExchange.doExecute(ShuffleExchange.scala:115)
  51.     at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:117)
  52.     at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:117)
  53.     at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:138)
  54.     at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
  55.     at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:135)
  56.     at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:116)
  57.     at org.apache.spark.sql.execution.InputAdapter.inputRDDs(WholeStageCodegenExec.scala:252)
  58.     at org.apache.spark.sql.execution.aggregate.HashAggregateExec.inputRDDs(HashAggregateExec.scala:141)
  59.     at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:386)
  60.     at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:117)
  61.     at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:117)
  62.     at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:138)
  63.     at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
  64.     at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:135)
  65.     at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:116)
  66.     at org.apache.spark.sql.execution.exchange.ShuffleExchange.prepareShuffleDependency(ShuffleExchange.scala:88)
  67.     at org.apache.spark.sql.execution.exchange.ShuffleExchange$$anonfun$doExecute$1.apply(ShuffleExchange.scala:124)
  68.     at org.apache.spark.sql.execution.exchange.ShuffleExchange$$anonfun$doExecute$1.apply(ShuffleExchange.scala:115)
  69.     at org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:52)
  70.     ... 35 more
  71. Caused by: org.apache.spark.sql.AnalysisException: java.lang.OutOfMemoryError: GC overhead limit exceeded;
  72.     at org.apache.spark.sql.hive.HiveExternalCatalog.withClient(HiveExternalCatalog.scala:106)
  73.     at org.apache.spark.sql.hive.HiveExternalCatalog.listPartitionsByFilter(HiveExternalCatalog.scala:1107)
  74.     at org.apache.spark.sql.hive.execution.HiveTableScanExec.rawPartitions$lzycompute(HiveTableScanExec.scala:170)
  75.     at org.apache.spark.sql.hive.execution.HiveTableScanExec.rawPartitions(HiveTableScanExec.scala:164)
  76.     at org.apache.spark.sql.hive.execution.HiveTableScanExec$$anonfun$11.apply(HiveTableScanExec.scala:192)
  77.     at org.apache.spark.sql.hive.execution.HiveTableScanExec$$anonfun$11.apply(HiveTableScanExec.scala:192)
  78.     at org.apache.spark.util.Utils$.withDummyCallSite(Utils.scala:2472)
  79.     at org.apache.spark.sql.hive.execution.HiveTableScanExec.doExecute(HiveTableScanExec.scala:191)
  80.     at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:117)
  81.     at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:117)
  82.     at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:138)
  83.     at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
  84.     at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:135)
  85.     at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:116)
  86.     at org.apache.spark.sql.execution.InputAdapter.inputRDDs(WholeStageCodegenExec.scala:252)
  87.     at org.apache.spark.sql.execution.aggregate.HashAggregateExec.inputRDDs(HashAggregateExec.scala:141)
  88.     at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:386)
  89.     at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:117)
  90.     at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:117)
  91.     at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:138)
  92.     at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
  93.     at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:135)
  94.     at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:116)
  95.     at org.apache.spark.sql.execution.exchange.ShuffleExchange.prepareShuffleDependency(ShuffleExchange.scala:88)
  96.     at org.apache.spark.sql.execution.exchange.ShuffleExchange$$anonfun$doExecute$1.apply(ShuffleExchange.scala:124)
  97.     at org.apache.spark.sql.execution.exchange.ShuffleExchange$$anonfun$doExecute$1.apply(ShuffleExchange.scala:115)
  98.     at org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:52)
  99.     ... 55 more
  100. Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded
  101.     at org.apache.hadoop.hive.metastore.api.StorageDescriptor.<init>(StorageDescriptor.java:256)
  102.     at org.apache.hadoop.hive.metastore.api.Partition.<init>(Partition.java:216)
  103.     at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.deepCopy(HiveMetaStoreClient.java:1563)
  104.     at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.deepCopyPartitions(HiveMetaStoreClient.java:1637)
  105.     at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.deepCopyPartitions(HiveMetaStoreClient.java:1625)
  106.     at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.listPartitions(HiveMetaStoreClient.java:1050)
  107.     at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
  108.     at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
  109.     at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  110.     at java.lang.reflect.Method.invoke(Method.java:498)
  111.     at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.invoke(RetryingMetaStoreClient.java:156)
  112.     at com.sun.proxy.$Proxy28.listPartitions(Unknown Source)
  113.     at org.apache.hadoop.hive.ql.metadata.Hive.getAllPartitionsOf(Hive.java:2096)
  114.     at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
  115.     at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
  116.     at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  117.     at java.lang.reflect.Method.invoke(Method.java:498)
  118.     at org.apache.spark.sql.hive.client.Shim_v0_13.getPartitionsByFilter(HiveShim.scala:628)
  119.     at org.apache.spark.sql.hive.client.HiveClientImpl$$anonfun$getPartitionsByFilter$1.apply(HiveClientImpl.scala:596)
  120.     at org.apache.spark.sql.hive.client.HiveClientImpl$$anonfun$getPartitionsByFilter$1.apply(HiveClientImpl.scala:594)
  121.     at org.apache.spark.sql.hive.client.HiveClientImpl$$anonfun$withHiveState$1.apply(HiveClientImpl.scala:291)
  122.     at org.apache.spark.sql.hive.client.HiveClientImpl.liftedTree1$1(HiveClientImpl.scala:232)
  123.     at org.apache.spark.sql.hive.client.HiveClientImpl.retryLocked(HiveClientImpl.scala:231)
  124.     at org.apache.spark.sql.hive.client.HiveClientImpl.withHiveState(HiveClientImpl.scala:274)
  125.     at org.apache.spark.sql.hive.client.HiveClientImpl.getPartitionsByFilter(HiveClientImpl.scala:594)
  126.     at org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$listPartitionsByFilter$1.apply(HiveExternalCatalog.scala:1114)
  127.     at org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$listPartitionsByFilter$1.apply(HiveExternalCatalog.scala:1107)
  128.     at org.apache.spark.sql.hive.HiveExternalCatalog.withClient(HiveExternalCatalog.scala:97)
  129.     at org.apache.spark.sql.hive.HiveExternalCatalog.listPartitionsByFilter(HiveExternalCatalog.scala:1107)
  130.     at org.apache.spark.sql.hive.execution.HiveTableScanExec.rawPartitions$lzycompute(HiveTableScanExec.scala:170)
  131.     at org.apache.spark.sql.hive.execution.HiveTableScanExec.rawPartitions(HiveTableScanExec.scala:164)
  132.     at org.apache.spark.sql.hive.execution.HiveTableScanExec$$anonfun$11.apply(HiveTableScanExec.scala:192)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!