set hive.tez.exec.print.summary=true;
SET hive.execution.engine=tez;
SET hive.tez.container.size=8192;
SET hive.tez.java.opts=-server -Xmx7000m -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps
set hive.tez.auto.reducer.parallelism=false;
SET tez.am.resource.memory.mb=8192;
select vins.cvdc62_calcd_vin_token_d, pii.vin_2
FROM
(Select vin_2, vin_token_d
FROM cvdp.ncvdpen_ncvde01_vin_pii_hte
WHERE last_updt_on_s IS NOT NULL
GROUP BY vin_2, vin_token_d) pii
JOIN (Select cvdc62_calcd_vin_token_d
From cvdp.ncvdc62_tcuftcp_msg_sec_hte WHERE
cvdc62_partition_region_x IS NOT NULL AND
cvdc62_partition_country_x IS NOT NULL AND
cvdc62_partition_date_x > '1990-01-01 00:00:00'
GROUP BY cvdc62_calcd_vin_token_d) vins
ON (pii.vin_token_d = vins.cvdc62_calcd_vin_token_d);
AM INPUTS
Vertex vertex_1497195879957_1886503_1_00
» Map 4
killed/failed due to:ROOT_INPUT_INIT_FAILURE
Vertex Input: ncvdc62_tcuftcp_msg_sec_hte initializer failed, vertex=vertex_1497195879957_1886503_1_00
» Map 4
, java.lang.RuntimeException: serious problem
at org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.generateSplitsInfo(OrcInputFormat.java:1273)
at org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.getSplits(OrcInputFormat.java:1300)
at org.apache.hadoop.hive.ql.io.HiveInputFormat.addSplitsForGroup(HiveInputFormat.java:307)
at org.apache.hadoop.hive.ql.io.HiveInputFormat.getSplits(HiveInputFormat.java:409)
at org.apache.hadoop.hive.ql.exec.tez.HiveSplitGenerator.initialize(HiveSplitGenerator.java:155)
at org.apache.tez.dag.app.dag.RootInputInitializerManager$InputInitializerCallable$1.run(RootInputInitializerManager.java:273)
at org.apache.tez.dag.app.dag.RootInputInitializerManager$InputInitializerCallable$1.run(RootInputInitializerManager.java:266)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1724)
at org.apache.tez.dag.app.dag.RootInputInitializerManager$InputInitializerCallable.call(RootInputInitializerManager.java:266)
at org.apache.tez.dag.app.dag.RootInputInitializerManager$InputInitializerCallable.call(RootInputInitializerManager.java:253)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.util.concurrent.ExecutionException: java.lang.OutOfMemoryError: Java heap space
at java.util.concurrent.FutureTask.report(FutureTask.java:122)
at java.util.concurrent.FutureTask.get(FutureTask.java:192)
at org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.generateSplitsInfo(OrcInputFormat.java:1268)
... 15 more
Caused by: java.lang.OutOfMemoryError: Java heap space
at com.google.protobuf.ByteString.copyFrom(ByteString.java:192)
at com.google.protobuf.CodedInputStream.readBytes(CodedInputStream.java:324)
at org.apache.hadoop.hive.ql.io.orc.OrcProto$StringStatistics.(OrcProto.java:1331)
at org.apache.hadoop.hive.ql.io.orc.OrcProto$StringStatistics.(OrcProto.java:1281)
at org.apache.hadoop.hive.ql.io.orc.OrcProto$StringStatistics$1.parsePartialFrom(OrcProto.java:1374)
at org.apache.hadoop.hive.ql.io.orc.OrcProto$StringStatistics$1.parsePartialFrom(OrcProto.java:1369)
at com.google.protobuf.CodedInputStream.readMessage(CodedInputStream.java:309)
at org.apache.hadoop.hive.ql.io.orc.OrcProto$ColumnStatistics.(OrcProto.java:4897)
at org.apache.hadoop.hive.ql.io.orc.OrcProto$ColumnStatistics.(OrcProto.java:4813)
at org.apache.hadoop.hive.ql.io.orc.OrcProto$ColumnStatistics$1.parsePartialFrom(OrcProto.java:5005)
at org.apache.hadoop.hive.ql.io.orc.OrcProto$ColumnStatistics$1.parsePartialFrom(OrcProto.java:5000)
at com.google.protobuf.CodedInputStream.readMessage(CodedInputStream.java:309)
at org.apache.hadoop.hive.ql.io.orc.OrcProto$StripeStatistics.(OrcProto.java:14334)
at org.apache.hadoop.hive.ql.io.orc.OrcProto$StripeStatistics.(OrcProto.java:14281)
at org.apache.hadoop.hive.ql.io.orc.OrcProto$StripeStatistics$1.parsePartialFrom(OrcProto.java:14370)
at org.apache.hadoop.hive.ql.io.orc.OrcProto$StripeStatistics$1.parsePartialFrom(OrcProto.java:14365)
at com.google.protobuf.CodedInputStream.readMessage(CodedInputStream.java:309)
at org.apache.hadoop.hive.ql.io.orc.OrcProto$Metadata.(OrcProto.java:15008)
at org.apache.hadoop.hive.ql.io.orc.OrcProto$Metadata.(OrcProto.java:14955)
at org.apache.hadoop.hive.ql.io.orc.OrcProto$Metadata$1.parsePartialFrom(OrcProto.java:15044)
at org.apache.hadoop.hive.ql.io.orc.OrcProto$Metadata$1.parsePartialFrom(OrcProto.java:15039)
at com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:89)
at com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:95)
at com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:49)
at org.apache.hadoop.hive.ql.io.orc.OrcProto$Metadata.parseFrom(OrcProto.java:15176)
at org.apache.hadoop.hive.ql.io.orc.ReaderImpl.extractMetadata(ReaderImpl.java:470)
at org.apache.hadoop.hive.ql.io.orc.OrcTail.getMetadata(OrcTail.java:131)
at org.apache.hadoop.hive.ql.io.orc.OrcTail.getStripeStatisticsProto(OrcTail.java:101)
at org.apache.hadoop.hive.ql.io.orc.OrcTail.getStripeStatistics(OrcTail.java:91)
at org.apache.hadoop.hive.ql.io.orc.OrcInputFormat$SplitGenerator.populateAndCacheStripeDetails(OrcInputFormat.java:1110)
at org.apache.hadoop.hive.ql.io.orc.OrcInputFormat$SplitGenerator.callInternal(OrcInputFormat.java:1001)
at org.apache.hadoop.hive.ql.io.orc.OrcInputFormat$SplitGenerator.access$2000(OrcInputFormat.java:838)
AUTO REDUCER
ERROR : DAG did not succeed due to VERTEX_FAILURE. failedVertices:1 killedVertices:3
Error: Error while processing statement: FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.tez.TezTask. Vertex failed, vertexName=Reducer 2, vertexId=vertex_1497195879957_1880423_1_03, diagnostics=[Exception in VertexManager, vertex:vertex_1497195879957_1880423_1_03 [Reducer 2],org.apache.tez.dag.api.TezUncheckedException: Atleast 1 bipartite source should exist
at org.apache.tez.dag.library.vertexmanager.ShuffleVertexManager.onVertexStarted(ShuffleVertexManager.java:456)
at org.apache.tez.dag.app.dag.impl.VertexManager$VertexManagerEventOnVertexStarted.invoke(VertexManager.java:587)
at org.apache.tez.dag.app.dag.impl.VertexManager$VertexManagerEvent$1.run(VertexManager.java:658)
at org.apache.tez.dag.app.dag.impl.VertexManager$VertexManagerEvent$1.run(VertexManager.java:653)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1724)
at org.apache.tez.dag.app.dag.impl.VertexManager$VertexManagerEvent.call(VertexManager.java:653)
at org.apache.tez.dag.app.dag.impl.VertexManager$VertexManagerEvent.call(VertexManager.java:642)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
, Vertex did not succeed due to AM_USERCODE_FAILURE, failedTasks:0 killedTasks:253, Vertex vertex_1497195879957_1880423_1_03 [Reducer 2] killed/failed due to:AM_USERCODE_FAILURE]Vertex killed, vertexName=Reducer 5, vertexId=vertex_1497195879957_1880423_1_01, diagnostics=[Vertex received Kill in INITED state., Vertex vertex_1497195879957_1880423_1_01 [Reducer 5] killed/failed due to:OTHER_VERTEX_FAILURE]Vertex killed, vertexName=Map 4, vertexId=vertex_1497195879957_1880423_1_00, diagnostics=[Vertex received Kill in INITED state., Vertex vertex_1497195879957_1880423_1_00 [Map 4] killed/failed due to:OTHER_VERTEX_FAILURE]Vertex killed, vertexName=Reducer 3, vertexId=vertex_1497195879957_1880423_1_04, diagnostics=[Vertex received Kill in INITED state., Vertex vertex_1497195879957_1880423_1_04 [Reducer 3] killed/failed due to:OTHER_VERTEX_FAILURE]DAG did not succeed due to VERTEX_FAILURE. failedVertices:1 killedVertices:3 (state=08S01,code=2)
