API tools faq
paste
josephxsxn

notes

josephxsxn
Feb 9th, 2018
128
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Bash 9.56 KB | None | 0 0
raw download clone embed print report
  1. CREATE EXTERNAL TABLE `outbound.partitioned_orc_test`(                                              
  2.    `id` string COMMENT '',                                                        
  3.    `messageid` string COMMENT '',                                                  
  4.    `streamname` string COMMENT '',                                                
  5.    `subscriptionname` string COMMENT '',                                          
  6.    `keyword` string COMMENT '',                                                    
  7.    `messagetype` string COMMENT '',                                                
  8.    `unixtime` bigint COMMENT '',                                                  
  9.    `mid` string COMMENT '',                                                        
  10.    `createddt` string COMMENT '',                                                  
  11.    `indexproviderid` string COMMENT '',                                            
  12.    `siteid` string COMMENT '',                                                    
  13.    `domain` string COMMENT '',                                                    
  14.    `lang` string COMMENT '',                                                      
  15.    `country` string COMMENT '',                                                    
  16.    `crawleddt` string COMMENT '',
  17.    `ip` string,
  18.    `day` int
  19. )                                                  
  20.  PARTITIONED BY (                                                                  
  21.    `customerid` int,                                                              
  22.    `providerid` int,                                                              
  23.    `year` int,                                                                    
  24.    `month` int                                                                    
  25. )
  26. stored as ORC;    
  27.  
  28. for i in {-1..543}
  29. do
  30. for y in {2015..2018}
  31. do
  32. for m in {1..12}
  33. do
  34. beeline -u "jdbc:hive2://hdp-prod-master02.boardreader.com:2181,hdp-prod-master01.boardreader.com:2181,hdp-prod-master03.boardreader.com:2181/;serviceDiscoveryMode=zooKeeper;zooKeeperNamespace=hiveserver2" -n root -p root \
  35. -e"
  36. set mapreduce.job.name='conversion job partition $i $y $m';
  37. set tez.task.log.level=ERROR;
  38. set tez.am.log.level=ERROR;
  39. set tez.grouping.min-size=16777216;
  40. set tez.grouping.max-size=512000000;
  41. set hive.exec.reducers.max=10000;
  42. set tez.am.resource.memory.mb=24000;
  43. set hive.exec.dynamic.partition.mode=nonstrict;
  44. set hive.exec.dynamic.partition=true;
  45. set hive.exec.max.dynamic.partitions=1000000000;
  46. set hive.tez.container.size=3096;
  47. set hive.tez.java.opts=-Xmx2500m -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps;
  48. INSERT OVERWRITE TABLE outbound.partitioned_orc_test PARTITION (customerid, providerid, year, month)
  49. SELECT id, messageid, streamname, subscriptionname, keyword, messagetype, unixtime, mid, createddt, indexproviderid, siteid, domain, lang, country, crawleddt, ip, day, customerid, providerid, year, month
  50. FROM outbound.partitioned
  51. WHERE customerid=$i AND year=$y AND month=$m
  52. DISTRIBUTE BY customerid, providerid, year, month;"
  53. done
  54. done
  55. done
  56.  
  57.  
  58.  
  59.  
  60. hbase org.apache.hadoop.hbase.mapreduce.RowCounter
  61.  
  62. 2018-02-09 14:55:06,159 [INFO] [AMRM Callback Handler Thread] |rm.YarnTaskSchedulerService|: Allocated: <memory:0, vCores:0> Free: <memory:688128, vCores:143> pendingRequests: 0 delayedContainers: 0 heartbeats: 2701 lastPreemptionHeartbeat: 2700
  63. 2018-02-09 14:55:56,841 [WARN] [ResponseProcessor for block BP-484199763-192.168.197.14-1444236781588:blk_1125819368_52080718] |hdfs.DFSClient|: DFSOutputStream ResponseProcessor exception  for block BP-484199763-192.168.197.14-1444236781588:blk_1125819368_52080718
  64. java.io.EOFException: Premature EOF: no length prefix available
  65.     at org.apache.hadoop.hdfs.protocolPB.PBHelper.vintPrefixed(PBHelper.java:2464)
  66.     at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244)
  67.     at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer$ResponseProcessor.run(DFSOutputStream.java:843)
  68. 2018-02-09 14:55:56,841 [WARN] [ResponseProcessor for block BP-484199763-192.168.197.14-1444236781588:blk_1125819369_52080719] |hdfs.DFSClient|: DFSOutputStream ResponseProcessor exception  for block BP-484199763-192.168.197.14-1444236781588:blk_1125819369_52080719
  69. java.io.EOFException: Premature EOF: no length prefix available
  70.     at org.apache.hadoop.hdfs.protocolPB.PBHelper.vintPrefixed(PBHelper.java:2464)
  71.     at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244)
  72.     at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer$ResponseProcessor.run(DFSOutputStream.java:843)
  73. 2018-02-09 14:55:56,844 [WARN] [DataStreamer for file /tmp/hive/hive/_tez_session_dir/720c7ac5-05b0-4f6d-b77a-be573ac27a1e/.tez/application_1518203762012_0005/recovery/1/summary block BP-484199763-192.168.197.14-1444236781588:blk_1125819368_52080718] |hdfs.DFSClient|: Error Recovery for block BP-484199763-192.168.197.14-1444236781588:blk_1125819368_52080718 in pipeline DatanodeInfoWithStorage[192.168.197.122:50010,DS-8c7c32b6-1df8-4f98-8c34-e0d5b98dfd53,DISK], DatanodeInfoWithStorage[192.168.197.114:50010,DS-64c48141-a07b-46ff-b272-1002c405a618,DISK], DatanodeInfoWithStorage[192.168.197.111:50010,DS-d132aa58-e22d-4d0c-804d-f03097c3496a,DISK]: bad datanode DatanodeInfoWithStorage[192.168.197.122:50010,DS-8c7c32b6-1df8-4f98-8c34-e0d5b98dfd53,DISK]
  74. 2018-02-09 14:55:56,844 [WARN] [DataStreamer for file /tmp/hive/hive/_tez_session_dir/720c7ac5-05b0-4f6d-b77a-be573ac27a1e/.tez/application_1518203762012_0005/recovery/1/dag_1518203762012_0005_1.recovery block BP-484199763-192.168.197.14-1444236781588:blk_1125819369_52080719] |hdfs.DFSClient|: Error Recovery for block BP-484199763-192.168.197.14-1444236781588:blk_1125819369_52080719 in pipeline DatanodeInfoWithStorage[192.168.197.122:50010,DS-8c7c32b6-1df8-4f98-8c34-e0d5b98dfd53,DISK], DatanodeInfoWithStorage[192.168.197.121:50010,DS-aa3c6c78-b9be-4cf3-b77c-847acdff008d,DISK], DatanodeInfoWithStorage[192.168.197.120:50010,DS-56de098a-13c2-4f66-b3a4-a5546371d5dd,DISK]: bad datanode DatanodeInfoWithStorage[192.168.197.122:50010,DS-8c7c32b6-1df8-4f98-8c34-e0d5b98dfd53,DISK]
  75. 2018-02-09 14:55:59,095 [INFO] [AMRM Callback Handler Thread] |rm.YarnTaskSchedulerService|: Allocated: <memory:0, vCores:0> Free: <memory:688128, vCores:143> pendingRequests: 0 delayedContainers: 0 heartbeats: 2751 lastPreemptionHeartbeat: 2750
  76. 2018-02-09 14:57:23,873 [INFO] [AMRM Callback Handler Thread] |rm.YarnTaskSchedulerService|: Allocated: <memory:0, vCores:0> Free: <memory:688128, vCores:143> pendingRequests: 0 delayedContainers: 0 heartbeats: 2801 lastPreemptionHeartbeat: 2800
  77.  
  78. 2018-02-09 14:55:56,841 [WARN] [ResponseProcessor for block BP-484199763-192.168.197.14-1444236781588:blk_1125819368_52080718] |hdfs.DFSClient|: DFSOutputStream ResponseProcessor exception  for block BP-484199763-192.168.197.14-1444236781588:blk_1125819368_52080718
  79. java.io.EOFException: Premature EOF: no length prefix available
  80.     at org.apache.hadoop.hdfs.protocolPB.PBHelper.vintPrefixed(PBHelper.java:2464)
  81.     at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244)
  82.     at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer$ResponseProcessor.run(DFSOutputStream.java:843)
  83. 2018-02-09 14:55:56,841 [WARN] [ResponseProcessor for block BP-484199763-192.168.197.14-1444236781588:blk_1125819369_52080719] |hdfs.DFSClient|: DFSOutputStream ResponseProcessor exception  for block BP-484199763-192.168.197.14-1444236781588:blk_1125819369_52080719
  84. java.io.EOFException: Premature EOF: no length prefix available
  85.     at org.apache.hadoop.hdfs.protocolPB.PBHelper.vintPrefixed(PBHelper.java:2464)
  86.     at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244)
  87.     at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer$ResponseProcessor.run(DFSOutputStream.java:843)
  88. 2018-02-09 14:55:56,844 [WARN] [DataStreamer for file /tmp/hive/hive/_tez_session_dir/720c7ac5-05b0-4f6d-b77a-be573ac27a1e/.tez/application_1518203762012_0005/recovery/1/summary block BP-484199763-192.168.197.14-1444236781588:blk_1125819368_52080718] |hdfs.DFSClient|: Error Recovery for block BP-484199763-192.168.197.14-1444236781588:blk_1125819368_52080718 in pipeline DatanodeInfoWithStorage[192.168.197.122:50010,DS-8c7c32b6-1df8-4f98-8c34-e0d5b98dfd53,DISK], DatanodeInfoWithStorage[192.168.197.114:50010,DS-64c48141-a07b-46ff-b272-1002c405a618,DISK], DatanodeInfoWithStorage[192.168.197.111:50010,DS-d132aa58-e22d-4d0c-804d-f03097c3496a,DISK]: bad datanode DatanodeInfoWithStorage[192.168.197.122:50010,DS-8c7c32b6-1df8-4f98-8c34-e0d5b98dfd53,DISK]
  89. 2018-02-09 14:55:56,844 [WARN] [DataStreamer for file /tmp/hive/hive/_tez_session_dir/720c7ac5-05b0-4f6d-b77a-be573ac27a1e/.tez/application_1518203762012_0005/recovery/1/dag_1518203762012_0005_1.recovery block BP-484199763-192.168.197.14-1444236781588:blk_1125819369_52080719] |hdfs.DFSClient|: Error Recovery for block BP-484199763-192.168.197.14-1444236781588:blk_1125819369_52080719 in pipeline DatanodeInfoWithStorage[192.168.197.122:50010,DS-8c7c32b6-1df8-4f98-8c34-e0d5b98dfd53,DISK], DatanodeInfoWithStorage[192.168.197.121:50010,DS-aa3c6c78-b9be-4cf3-b77c-847acdff008d,DISK], DatanodeInfoWithStorage[192.168.197.120:50010,DS-56de098a-13c2-4f66-b3a4-a5546371d5dd,DISK]: bad datanode DatanodeInfoWithStorage[192.168.197.122:50010,DS-8c7c32b6-1df8-4f98-8c34-e0d5b98dfd53,DISK]
  90. 2018-02-09 14:55:59,095 [INFO] [AMRM Callback Handler Thread] |rm.YarnTaskSchedulerService|: Allocated: <memory:0, vCores:0> Free: <memory:688128, vCores:143> pendingRequests: 0 delayedContainers: 0 heartbeats: 2751 lastPreemptionHeartbeat: 2750
Add Comment
Please, Sign In to add comment
Public Pastes
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!