View difference between Paste ID: 8YbDg1cf and k1RzvNjf
SHOW: | | - or go back to the newest paste.
1
************** HIVE **********************
2
CREATE EXTERNAL TABLE `customer`(
3
  `id` int,
4
  `first` string,
5
  `last` string)
6
ROW FORMAT SERDE
7
  'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
8
WITH SERDEPROPERTIES (
9
  'field.delim'=',',
10
  'serialization.format'=',')
11
STORED AS INPUTFORMAT
12
  'org.apache.hadoop.mapred.TextInputFormat'
13
OUTPUTFORMAT
14
  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
15
LOCATION
16
  's3a://testbucket/'
17
TBLPROPERTIES (
18
  'transient_lastDdlTime'='1524321660')
19
20
hive> use minio;
21
OK
22
Time taken: 2.398 seconds
23
24
hive> select * from customer;
25
OK
26
5       Bob     Jones
27
6       Phil    Brune
28
Time taken: 3.359 seconds, Fetched: 2 row(s)
29
30
************** PRESTO **********************
31
32
presto:minio> select * from customer;
33
34
Query 20180422_102158_00007_7etq4, FAILED, 1 node
35
Splits: 16 total, 0 done (0.00%)
36
0:38 [0 rows, 0B] [0 rows/s, 0B/s]
37
38
Query 20180422_102158_00007_7etq4 failed: Unable to execute HTTP request: testbucket.minio: Name or service not known
39
40
connector.name=hive-hadoop2
41
hive.metastore.uri=thrift://hivemeta:9083
42
hive.s3.endpoint=http://minio:9000
43
hive.s3.aws-access-key=OQ4K2YFH7S1BE6ZM10M0
44-
hive.s3.aws-secret-key=Ft5iKPkY0YZNoDpp6r5HgbjFuGAxn+jt1avBPyHq
44+
hive.s3.aws-secret-key=Ft5iKPkY0YZNoDpp6r5HgbjFuGAxn+jt1avBPyHq
45
46
Presto server.log:
47
2018-04-22T06:22:36.751-0400    INFO    hive-minio-1    com.amazonaws.latency   ServiceName=[Amazon S3], ServiceEndpoint=[http://testbucket.minio:9000], Exception=[java.net.UnknownHostException: testbucket.minio: Name or service not known, java.net.UnknownHostException: testbucket.minio, java.net.UnknownHostException: testbucket.minio, java.net.UnknownHostException: testbucket.minio, java.net.UnknownHostException: testbucket.minio, java.net.UnknownHostException: testbucket.minio, java.net.UnknownHostException: testbucket.minio, java.net.UnknownHostException: testbucket.minio, java.net.UnknownHostException: testbucket.minio, java.net.UnknownHostException: testbucket.minio: Name or service not known, java.net.UnknownHostException: testbucket.minio: Name or service not known], RequestType=[ListObjectsRequest], AWSRequestID=[null, null, null, null, null, null, null, null, null, null, null], HttpClientPoolPendingCount=0, RetryCapacityConsumed=50, HttpClientPoolAvailableCount=0, RequestCount=11, Exception=11, HttpClientPoolLeasedCount=0, RetryPauseTime=[71.689, 121.241, 316.824, 701.24, 1265.152, 144.187, 286.111, 1902.218, 16082.208, 14961.288], ClientExecuteTime=[36048.634], HttpRequestTime=[55.65, 0.973, 0.693, 0.599, 0.769, 0.75, 0.664, 2.385, 0.736, 14.755, 14.781], RequestSigningTime=[71.799, 0.977, 0.459, 0.569, 0.585, 0.565, 0.415, 0.521, 0.789, 0.472, 0.642], CredentialsRequestTime=[0.035, 0.002, 0.004, 0.003, 0.003, 0.002, 0.003, 0.002, 0.002, 0.002, 0.003, 0.002],
48
2018-04-22T06:22:36.884-0400    INFO    query-execution-1       com.facebook.presto.event.query.QueryMonitor    TIMELINE: Query 20180422_102158_00007_7etq4 :: Transaction:[fcb2c776-3c86-4449-a2e2-61e67e670cd2] :: elapsed 37787ms :: planning 858ms :: scheduling 36929ms :: running 0ms :: finishing 36929ms :: begin 2018-04-22T06:21:58.970-04:00 :: end 2018-04-22T06:22:36.757-04:00
49
50
51
52
53
core-site.xml
54
<property>
55
   <name>fs.s3a.connection.ssl.enabled</name>
56
   <value>false</value>
57
   <description>Enables or disables SSL connections to S3.</description>
58
</property>
59
60
<property>
61
 <name>fs.s3a.endpoint</name>
62
 <description>AWS S3 endpoint to connect to. An up-to-date list is
63
  provided in the AWS Documentation: regions and endpoints. Without this
64
  property, the standard region (s3.amazonaws.com) is assumed.
65
 </description>
66
 <value>http://minio:9000</value>
67
</property>
68
69
<property>
70
<name>fs.s3a.awsAccessKeyId</name>
71
<description>AWS access key ID.</description>
72
<value>OQ4K2YFH7S1BE6ZM10M0</value>
73
</property>
74
75
<property>
76
<name>fs.s3a.access.key</name>
77
<description>AWS access key ID.</description>
78
<value>OQ4K2YFH7S1BE6ZM10M0</value>
79
</property>
80
81
82
<property>
83
  <name>fs.s3a.awsSecretAccessKey</name>
84
  <description>AWS secret key.</description>
85
  <value>Ft5iKPkY0YZNoDpp6r5HgbjFuGAxn+jt1avBPyHq</value>
86
</property>
87
88
<property>
89
  <name>fs.s3a.secret.key</name>
90
  <description>AWS secret key.</description>
91
  <value>Ft5iKPkY0YZNoDpp6r5HgbjFuGAxn+jt1avBPyHq</value>
92
</property>
93
94
<property>
95
<name>fs.s3a.path.style.access</name>
96
<value>true</value>
97
<description>Enable S3 path style access ie disabling the default virtual hosting behaviour.
98
  Useful for S3A-compliant storage providers as it removes the need to set up DNS for virtual hosting.
99
</description>
100
</property>
101
102
<property>
103
<name>fs.s3a.impl</name>
104
<value>org.apache.hadoop.fs.s3a.S3AFileSystem</value>
105
<description>The implementation class of the S3A Filesystem</description>
106
</property>