Advertisement
Guest User

Untitled

a guest
Jun 17th, 2018
129
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Java 1.50 KB | None | 0 0
  1.         conf = new SparkConf()
  2. //                .setMaster("local")
  3. //                .set("spark.driver.allowMultipleContexts", "true")
  4.                 .setMaster("spark://10.2.28.19:7077")
  5.                 .set("spark.ui.port", "8080")
  6.                 .setSparkHome("/opt/spark-2.3.0-bin-hadoop2.7/bin")
  7.                 .set("spark.driver.allowMultipleContexts", "true")
  8.                 .set("spark.executor.host", "10.2.28.31")
  9.                 .set("spark.driver.bindAddress", "10.2.28.31")
  10.                 .set("spark.driver.host", "10.2.28.31")
  11. //                .setJars(new String[] { "out/sparkapp.jar" })
  12.                 .setAppName("Spark Classifiers");
  13.  
  14.     public void connect() {
  15.  
  16.         /* Dla Sparka */
  17.         rowDataset = session.read()
  18.                 .option("driver", Constants.JDBC_DRIVER)
  19.                 .option("url", Constants.DB_URL)
  20.                 .option("dbtable", dbtable)
  21.                 .option("user", Constants.DB_USER)
  22.                 .option("password", Constants.DB_PASS)
  23.                 .option("inferSchema", true)
  24.                 .format(Constants.JDBC_SPARK_DRIVER)
  25.                 .load();
  26.  
  27.  
  28.  public static final String DB_URL  = "jdbc:postgresql://10.2.28.19:5432/postgres";
  29.  public static final String DB_USER = "postgres";
  30.  public static final String DB_PASS = "postgres";
  31.     public static final String JDBC_DRIVER = "com.mysql.cj.jdbc.Driver";
  32.     public static final String JDBC_SPARK_DRIVER = "org.apache.spark.sql.execution.datasources.jdbc.DefaultSource";
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement