Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- sqoop-import --connect "jdbc:mysql://quickstart.cloudera:3306/retail_db" \
- --username retail_dba --password cloudera --table products \
- --target-dir "/user/cloudera/products" \
- --fields-terminated-by '|'
- hadoop fs -mkdir problem2
- hadoop fs -mkdir problem2/products
- hadoop fs -mv products problem2
- hadoop fs -chmod 765 problem2/products
- from pyspark import Row
- productsData=sc.textFile("/user/cloudera/problem2/products")
- productsFilterd = productsData.filter(lambda rec: float(rec.split("|")[4])<100)
- products=productsFiltered.map(lambda rec:Row(product_id=int(rec.split("|")[0]),product_cat_id=int(rec.split("|")[1]),product_name=rec.split("|")[2],product_desc=rec.split("|")[3],product_price=float(rec.split("|")[4]),product_image=rec.split("|")[5])).toDF()
- products.registerTempTable("products")
- sqlContext.setConf("spark.sql.shuffle.partitions","2")
- sqlResult=sqlContext.sql("select product_cat_id,max(product_price) max_price,count(product_id) total_products, min(product_price) min_price,avg(product_price) from products group by product_cat_id sort by product_cat_id")
- sqlContext.setConf("spark.sql.avro.compression.codec","snappy")
- sqlResult.write.format("com.databricks.spark.avro").save("/user/cloudera/problem2/products/result-sql")
Add Comment
Please, Sign In to add comment