SHOW:
|
|
- or go back to the newest paste.
1 | ************** HIVE ********************** | |
2 | CREATE EXTERNAL TABLE `customer`( | |
3 | `id` int, | |
4 | `first` string, | |
5 | `last` string) | |
6 | ROW FORMAT SERDE | |
7 | 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' | |
8 | WITH SERDEPROPERTIES ( | |
9 | 'field.delim'=',', | |
10 | 'serialization.format'=',') | |
11 | STORED AS INPUTFORMAT | |
12 | 'org.apache.hadoop.mapred.TextInputFormat' | |
13 | OUTPUTFORMAT | |
14 | 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' | |
15 | LOCATION | |
16 | 's3a://testbucket/' | |
17 | TBLPROPERTIES ( | |
18 | 'transient_lastDdlTime'='1524321660') | |
19 | ||
20 | hive> use minio; | |
21 | OK | |
22 | Time taken: 2.398 seconds | |
23 | ||
24 | hive> select * from customer; | |
25 | OK | |
26 | 5 Bob Jones | |
27 | 6 Phil Brune | |
28 | Time taken: 3.359 seconds, Fetched: 2 row(s) | |
29 | ||
30 | ************** PRESTO ********************** | |
31 | ||
32 | presto:minio> select * from customer; | |
33 | ||
34 | Query 20180422_102158_00007_7etq4, FAILED, 1 node | |
35 | Splits: 16 total, 0 done (0.00%) | |
36 | 0:38 [0 rows, 0B] [0 rows/s, 0B/s] | |
37 | ||
38 | Query 20180422_102158_00007_7etq4 failed: Unable to execute HTTP request: testbucket.minio: Name or service not known | |
39 | ||
40 | connector.name=hive-hadoop2 | |
41 | hive.metastore.uri=thrift://hivemeta:9083 | |
42 | hive.s3.endpoint=http://minio:9000 | |
43 | hive.s3.aws-access-key=OQ4K2YFH7S1BE6ZM10M0 | |
44 | - | hive.s3.aws-secret-key=Ft5iKPkY0YZNoDpp6r5HgbjFuGAxn+jt1avBPyHq |
44 | + | hive.s3.aws-secret-key=Ft5iKPkY0YZNoDpp6r5HgbjFuGAxn+jt1avBPyHq |
45 | ||
46 | Presto server.log: | |
47 | 2018-04-22T06:22:36.751-0400 INFO hive-minio-1 com.amazonaws.latency ServiceName=[Amazon S3], ServiceEndpoint=[http://testbucket.minio:9000], Exception=[java.net.UnknownHostException: testbucket.minio: Name or service not known, java.net.UnknownHostException: testbucket.minio, java.net.UnknownHostException: testbucket.minio, java.net.UnknownHostException: testbucket.minio, java.net.UnknownHostException: testbucket.minio, java.net.UnknownHostException: testbucket.minio, java.net.UnknownHostException: testbucket.minio, java.net.UnknownHostException: testbucket.minio, java.net.UnknownHostException: testbucket.minio, java.net.UnknownHostException: testbucket.minio: Name or service not known, java.net.UnknownHostException: testbucket.minio: Name or service not known], RequestType=[ListObjectsRequest], AWSRequestID=[null, null, null, null, null, null, null, null, null, null, null], HttpClientPoolPendingCount=0, RetryCapacityConsumed=50, HttpClientPoolAvailableCount=0, RequestCount=11, Exception=11, HttpClientPoolLeasedCount=0, RetryPauseTime=[71.689, 121.241, 316.824, 701.24, 1265.152, 144.187, 286.111, 1902.218, 16082.208, 14961.288], ClientExecuteTime=[36048.634], HttpRequestTime=[55.65, 0.973, 0.693, 0.599, 0.769, 0.75, 0.664, 2.385, 0.736, 14.755, 14.781], RequestSigningTime=[71.799, 0.977, 0.459, 0.569, 0.585, 0.565, 0.415, 0.521, 0.789, 0.472, 0.642], CredentialsRequestTime=[0.035, 0.002, 0.004, 0.003, 0.003, 0.002, 0.003, 0.002, 0.002, 0.002, 0.003, 0.002], | |
48 | 2018-04-22T06:22:36.884-0400 INFO query-execution-1 com.facebook.presto.event.query.QueryMonitor TIMELINE: Query 20180422_102158_00007_7etq4 :: Transaction:[fcb2c776-3c86-4449-a2e2-61e67e670cd2] :: elapsed 37787ms :: planning 858ms :: scheduling 36929ms :: running 0ms :: finishing 36929ms :: begin 2018-04-22T06:21:58.970-04:00 :: end 2018-04-22T06:22:36.757-04:00 | |
49 | ||
50 | ||
51 | ||
52 | ||
53 | core-site.xml | |
54 | <property> | |
55 | <name>fs.s3a.connection.ssl.enabled</name> | |
56 | <value>false</value> | |
57 | <description>Enables or disables SSL connections to S3.</description> | |
58 | </property> | |
59 | ||
60 | <property> | |
61 | <name>fs.s3a.endpoint</name> | |
62 | <description>AWS S3 endpoint to connect to. An up-to-date list is | |
63 | provided in the AWS Documentation: regions and endpoints. Without this | |
64 | property, the standard region (s3.amazonaws.com) is assumed. | |
65 | </description> | |
66 | <value>http://minio:9000</value> | |
67 | </property> | |
68 | ||
69 | <property> | |
70 | <name>fs.s3a.awsAccessKeyId</name> | |
71 | <description>AWS access key ID.</description> | |
72 | <value>OQ4K2YFH7S1BE6ZM10M0</value> | |
73 | </property> | |
74 | ||
75 | <property> | |
76 | <name>fs.s3a.access.key</name> | |
77 | <description>AWS access key ID.</description> | |
78 | <value>OQ4K2YFH7S1BE6ZM10M0</value> | |
79 | </property> | |
80 | ||
81 | ||
82 | <property> | |
83 | <name>fs.s3a.awsSecretAccessKey</name> | |
84 | <description>AWS secret key.</description> | |
85 | <value>Ft5iKPkY0YZNoDpp6r5HgbjFuGAxn+jt1avBPyHq</value> | |
86 | </property> | |
87 | ||
88 | <property> | |
89 | <name>fs.s3a.secret.key</name> | |
90 | <description>AWS secret key.</description> | |
91 | <value>Ft5iKPkY0YZNoDpp6r5HgbjFuGAxn+jt1avBPyHq</value> | |
92 | </property> | |
93 | ||
94 | <property> | |
95 | <name>fs.s3a.path.style.access</name> | |
96 | <value>true</value> | |
97 | <description>Enable S3 path style access ie disabling the default virtual hosting behaviour. | |
98 | Useful for S3A-compliant storage providers as it removes the need to set up DNS for virtual hosting. | |
99 | </description> | |
100 | </property> | |
101 | ||
102 | <property> | |
103 | <name>fs.s3a.impl</name> | |
104 | <value>org.apache.hadoop.fs.s3a.S3AFileSystem</value> | |
105 | <description>The implementation class of the S3A Filesystem</description> | |
106 | </property> |