Guest User

Untitled

a guest
Mar 8th, 2018
111
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 1.46 KB | None | 0 0
  1. val spark = SparkSession.builder().master("local[2]").appName("interfacing spark sql to hive metastore without configuration file")
  2. .config("hive.metastore.warehouse.dir", "C:\xxxx\xxxx\xxxx\")
  3. .enableHiveSupport() // don't forget to enable hive support
  4. .getOrCreate()
  5.  
  6. val sc = spark.sparkContext
  7. val sqlContext = spark.sqlContext
  8. val driverName = "org.apache.hive.jdbc.HiveDriver"
  9.  
  10. System.setProperty("javax.net.ssl.trustStore", "C:\xxxx\xxxx\xxxx\xxxx\xxxx\security\jssecacerts")
  11. System.setProperty("java.security.krb5.debug","true")
  12. System.setProperty("java.security.krb5.conf",new File("C:\xxxx\xxxx\krb5.conf").getAbsolutePath)
  13. System.setProperty("javax.security.auth.useSubjectCredsOnly", "false")
  14. System.setProperty("java.security.auth.login.config", new File("C:\xxxx\xxxx\jaas.conf").getAbsolutePath)
  15. val hiveurl="jdbc:hive2://xxxxx.octorp.com:10000/devl_dkp;user=pcpdosr;password=Kopdevp1;ssl=true;AuthMech=3"
  16. //;mapred.job.queue.name=dkl"
  17. val connectionProperties = new java.util.Properties()
  18.  
  19. sc.setLocalProperty("spark.scheduler.pool", "dkl")
  20. val hiveQuery = "select * from devl_dkp.employee"
  21.  
  22. val hiveResult = spark.read.option("driver",driverName).jdbc(hiveurl, hiveQuery, connectionProperties).collect()
  23.  
  24. exception caught: org.apache.hive.service.cli.HiveSQLException: Error while compiling statement: FAILED: ParseException line 1:14 cannot recognize input near 'select' '*' 'from' in join source
Add Comment
Please, Sign In to add comment