Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- #!/bin/bash
- # load the jar in hive server with the follwing commands
- # add the JAR in hive node
- #docker run --rm -v $PWD:/workdir -w /workdir -e HADOOP_USER_NAME=homeaway sequenceiq/hadoop-docker /usr/local/hadoop-2.7.0/bin/hdfs dfs -put /workdir/json-serde-1.3.8-jar-with-dependencies.jar hdfs://asthad011.wvrgroup.internal/apps/hive/warehouse/content_catalog_data
- # from hive command line do
- #ADD JAR hdfs://asthad011.wvrgroup.internal/apps/hive/warehouse/content_catalog_data/json-serde-1.3.8-jar-with-dependencies.jar;
- ######################## SEOTEXT CONTENT TABLE CREATION ######################################################
- echo "creating TEST image content table for Developer testing"
- docker run --rm \
- -it akanto/beeline \
- -u "jdbc:hive2://asthad010.wvrgroup.internal:10000/content_catalog?hive.tez.java.opts=-Xmx7650m;hive.tez.container.size=7650;tez.queue.name=product" \
- --debug \
- -n homeaway -e \
- 'CREATE TABLE content_catalog.image_contents_prod_json_vivek (
- id string,
- url string,
- etag string,
- inserted struct<dateString:string>,
- updated struct<dateString:string>,
- contentlocation string,
- contentsource string,
- contenttype string,
- flaggedforreview boolean,
- geoshape struct<coordinates:array<double>, type:string>,
- lbsid string,
- tags array<string>,
- classifiers array<string>,
- image struct<resource:string>
- )
- ROW FORMAT SERDE "org.apache.hive.hcatalog.data.JsonSerDe"
- WITH SERDEPROPERTIES ("mapping.id" = "_id");'
- ###################################################################################################
- echo -e "\n\nGoing to upload imageContent data into HDFS"
- docker run --rm -v $PWD:/workdir -w /workdir -e HADOOP_USER_NAME=homeaway sequenceiq/hadoop-docker /usr/local/hadoop-2.7.0/bin/hdfs dfs -put /workdir/vivek_data.json hdfs://asthad011.wvrgroup.internal/apps/hive/warehouse/content_catalog_data
- echo -e "Done uploading imageContent data into HDFS\n\n"
- # Push HDFS Data to CSV Backed Hive Tables
- echo -e "Going to upload imageContent data from HDFS to HIVE"
- docker run --rm akanto/beeline -u "jdbc:hive2://asthad010.wvrgroup.internal:10000/content_catalog?hive.tez.java.opts=-Xmx7650m;hive.tez.container.size=7650;tez.queue.name=product" -n homeaway -e "LOAD DATA INPATH \"/apps/hive/warehouse/content_catalog_data/vivek_data.json\" OVERWRITE INTO TABLE content_catalog.image_contents_prod_json_vivek;"
- echo -e "Done uploading imageContent data from HDFS to HIVE\n\n"
- echo -e "Going to query data in hive"
- docker run --rm -it akanto/beeline -u "jdbc:hive2://asthad010.wvrgroup.internal:10000/content_catalog?hive.tez.java.opts=-Xmx7650m;hive.tez.container.size=7650;tez.queue.name=product" -n homeaway -e "select * from image_contents_prod_json_vivek;"
- echo -e "Taking a look on data"
Add Comment
Please, Sign In to add comment