from pyspark.sql import SparkSession
from pyspark.sql import Row
warehouse_location = 'hdfs://quickstart.cloudera:8020/user/hive/warehouse'

spark = SparkSession
    .builder
    .appName("Python Spark SQL Hive integration example")
    .config("spark.sql.warehouse.dir", warehouse_location)
    .enableHiveSupport()
    .getOrCreate()

spark.sql("use qadb")

<property>
    <name>javax.jdo.option.ConnectionURL</name>
    <value>jdbc:mysql://quickstart.cloudera:8020/metastore?createDatabaseIfNotExist=true</value>
    <description>JDBC connect string for a JDBC metastore</description>
  </property>

  <property>
    <name>javax.jdo.option.ConnectionDriverName</name>
    <value>com.mysql.jdbc.Driver</value>
    <description>Driver class name for a JDBC metastore</description>
  </property>

  <property>
    <name>javax.jdo.option.ConnectionUserName</name>
    <value>hive</value>
  </property>

  <property>
    <name>javax.jdo.option.ConnectionPassword</name>
    <value>cloudera</value>
  </property>

  <property>
    <name>hive.hwi.war.file</name>
    <value>/usr/lib/hive/lib/hive-hwi-0.8.1-cdh4.0.0.jar</value>
    <description>This is the WAR file with the jsp content for Hive Web Interface</description>
  </property>

  <property>
    <name>datanucleus.fixedDatastore</name>
    <value>true</value>
  </property>

  <property>
    <name>datanucleus.autoCreateSchema</name>
    <value>false</value>
  </property>

  <property>
    <name>hive.metastore.uris</name>
    <value>thrift://quickstart.cloudera:8020</value>
    <description>IP address (or fully-qualified domain name) and port of the metastore host</description>
  </property>