Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- conf = new SparkConf()
- // .setMaster("local")
- // .set("spark.driver.allowMultipleContexts", "true")
- .setMaster("spark://10.2.28.19:7077")
- .set("spark.ui.port", "8080")
- .setSparkHome("/opt/spark-2.3.0-bin-hadoop2.7/bin")
- .set("spark.driver.allowMultipleContexts", "true")
- .set("spark.executor.host", "10.2.28.31")
- .set("spark.driver.bindAddress", "10.2.28.31")
- .set("spark.driver.host", "10.2.28.31")
- // .setJars(new String[] { "out/sparkapp.jar" })
- .setAppName("Spark Classifiers");
- public void connect() {
- /* Dla Sparka */
- rowDataset = session.read()
- .option("driver", Constants.JDBC_DRIVER)
- .option("url", Constants.DB_URL)
- .option("dbtable", dbtable)
- .option("user", Constants.DB_USER)
- .option("password", Constants.DB_PASS)
- .option("inferSchema", true)
- .format(Constants.JDBC_SPARK_DRIVER)
- .load();
- public static final String DB_URL = "jdbc:postgresql://10.2.28.19:5432/postgres";
- public static final String DB_USER = "postgres";
- public static final String DB_PASS = "postgres";
- public static final String JDBC_DRIVER = "com.mysql.cj.jdbc.Driver";
- public static final String JDBC_SPARK_DRIVER = "org.apache.spark.sql.execution.datasources.jdbc.DefaultSource";
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement