Advertisement
tdogchairropecup

Untitled

Apr 22nd, 2018
91
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. ************** HIVE **********************
  2. CREATE EXTERNAL TABLE `customer`(
  3. `id` int,
  4. `first` string,
  5. `last` string)
  6. ROW FORMAT SERDE
  7. 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
  8. WITH SERDEPROPERTIES (
  9. 'field.delim'=',',
  10. 'serialization.format'=',')
  11. STORED AS INPUTFORMAT
  12. 'org.apache.hadoop.mapred.TextInputFormat'
  13. OUTPUTFORMAT
  14. 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
  15. LOCATION
  16. 's3a://testbucket/'
  17. TBLPROPERTIES (
  18. 'transient_lastDdlTime'='1524321660')
  19.  
  20. hive> use minio;
  21. OK
  22. Time taken: 2.398 seconds
  23.  
  24. hive> select * from customer;
  25. OK
  26. 5 Bob Jones
  27. 6 Phil Brune
  28. Time taken: 3.359 seconds, Fetched: 2 row(s)
  29.  
  30. ************** PRESTO **********************
  31.  
  32. presto:minio> select * from customer;
  33.  
  34. Query 20180422_102158_00007_7etq4, FAILED, 1 node
  35. Splits: 16 total, 0 done (0.00%)
  36. 0:38 [0 rows, 0B] [0 rows/s, 0B/s]
  37.  
  38. Query 20180422_102158_00007_7etq4 failed: Unable to execute HTTP request: testbucket.minio: Name or service not known
  39.  
  40. connector.name=hive-hadoop2
  41. hive.metastore.uri=thrift://hivemeta:9083
  42. hive.s3.endpoint=http://minio:9000
  43. hive.s3.aws-access-key=OQ4K2YFH7S1BE6ZM10M0
  44. hive.s3.aws-secret-key=Ft5iKPkY0YZNoDpp6r5HgbjFuGAxn+jt1avBPyHq
  45.  
  46. Presto server.log:
  47. 2018-04-22T06:22:36.751-0400 INFO hive-minio-1 com.amazonaws.latency ServiceName=[Amazon S3], ServiceEndpoint=[http://testbucket.minio:9000], Exception=[java.net.UnknownHostException: testbucket.minio: Name or service not known, java.net.UnknownHostException: testbucket.minio, java.net.UnknownHostException: testbucket.minio, java.net.UnknownHostException: testbucket.minio, java.net.UnknownHostException: testbucket.minio, java.net.UnknownHostException: testbucket.minio, java.net.UnknownHostException: testbucket.minio, java.net.UnknownHostException: testbucket.minio, java.net.UnknownHostException: testbucket.minio, java.net.UnknownHostException: testbucket.minio: Name or service not known, java.net.UnknownHostException: testbucket.minio: Name or service not known], RequestType=[ListObjectsRequest], AWSRequestID=[null, null, null, null, null, null, null, null, null, null, null], HttpClientPoolPendingCount=0, RetryCapacityConsumed=50, HttpClientPoolAvailableCount=0, RequestCount=11, Exception=11, HttpClientPoolLeasedCount=0, RetryPauseTime=[71.689, 121.241, 316.824, 701.24, 1265.152, 144.187, 286.111, 1902.218, 16082.208, 14961.288], ClientExecuteTime=[36048.634], HttpRequestTime=[55.65, 0.973, 0.693, 0.599, 0.769, 0.75, 0.664, 2.385, 0.736, 14.755, 14.781], RequestSigningTime=[71.799, 0.977, 0.459, 0.569, 0.585, 0.565, 0.415, 0.521, 0.789, 0.472, 0.642], CredentialsRequestTime=[0.035, 0.002, 0.004, 0.003, 0.003, 0.002, 0.003, 0.002, 0.002, 0.002, 0.003, 0.002],
  48. 2018-04-22T06:22:36.884-0400 INFO query-execution-1 com.facebook.presto.event.query.QueryMonitor TIMELINE: Query 20180422_102158_00007_7etq4 :: Transaction:[fcb2c776-3c86-4449-a2e2-61e67e670cd2] :: elapsed 37787ms :: planning 858ms :: scheduling 36929ms :: running 0ms :: finishing 36929ms :: begin 2018-04-22T06:21:58.970-04:00 :: end 2018-04-22T06:22:36.757-04:00
  49.  
  50.  
  51.  
  52.  
  53. core-site.xml
  54. <property>
  55. <name>fs.s3a.connection.ssl.enabled</name>
  56. <value>false</value>
  57. <description>Enables or disables SSL connections to S3.</description>
  58. </property>
  59.  
  60. <property>
  61. <name>fs.s3a.endpoint</name>
  62. <description>AWS S3 endpoint to connect to. An up-to-date list is
  63. provided in the AWS Documentation: regions and endpoints. Without this
  64. property, the standard region (s3.amazonaws.com) is assumed.
  65. </description>
  66. <value>http://minio:9000</value>
  67. </property>
  68.  
  69. <property>
  70. <name>fs.s3a.awsAccessKeyId</name>
  71. <description>AWS access key ID.</description>
  72. <value>OQ4K2YFH7S1BE6ZM10M0</value>
  73. </property>
  74.  
  75. <property>
  76. <name>fs.s3a.access.key</name>
  77. <description>AWS access key ID.</description>
  78. <value>OQ4K2YFH7S1BE6ZM10M0</value>
  79. </property>
  80.  
  81.  
  82. <property>
  83. <name>fs.s3a.awsSecretAccessKey</name>
  84. <description>AWS secret key.</description>
  85. <value>Ft5iKPkY0YZNoDpp6r5HgbjFuGAxn+jt1avBPyHq</value>
  86. </property>
  87.  
  88. <property>
  89. <name>fs.s3a.secret.key</name>
  90. <description>AWS secret key.</description>
  91. <value>Ft5iKPkY0YZNoDpp6r5HgbjFuGAxn+jt1avBPyHq</value>
  92. </property>
  93.  
  94. <property>
  95. <name>fs.s3a.path.style.access</name>
  96. <value>true</value>
  97. <description>Enable S3 path style access ie disabling the default virtual hosting behaviour.
  98. Useful for S3A-compliant storage providers as it removes the need to set up DNS for virtual hosting.
  99. </description>
  100. </property>
  101.  
  102. <property>
  103. <name>fs.s3a.impl</name>
  104. <value>org.apache.hadoop.fs.s3a.S3AFileSystem</value>
  105. <description>The implementation class of the S3A Filesystem</description>
  106. </property>
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement