Advertisement
Guest User

Spark job submitting

a guest
Jun 28th, 2015
255
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Scala 1.09 KB | None | 0 0
  1. Configuration config = new Configuration();
  2. config.set("hdp.version", "2.2.4.2-2");
  3.  
  4. // fill properties from core-site.xml and yarn-site.xml to read data from hdfs
  5. fillProperties(config, getPropXmlAsMap("/etc/hadoop/conf/core-site.xml"));
  6. fillProperties(config, getPropXmlAsMap("/etc/hadoop/conf/yarn-site.xml"));
  7.  
  8. // spark configuration
  9. SparkConf sparkConf = new SparkConf();
  10. sparkConf.setMaster("yarn-cluster");
  11. sparkConf.set("hdp.version", "2.2.4.2-2");
  12.  
  13. // run args
  14. List<String> runArgs = Arrays.asList(
  15.    "--num-executors", "2",
  16.    "--executor-memory", "300M",
  17.    "--executor-cores", "1",
  18.    "--class", "job_class_name",
  19.    "--addJars", "hdfs://host:port/user/hdfs/spark-assembly-1.2.1.2.2.4.2-2-hadoop2.6.0.2.2.4.2-2.jar", // previously uploaded
  20.    "--jar", "hdfs://host:port/user/hdfs/application.jar",
  21.    "--arg", ".." // arg that passed to main method
  22. );
  23.  
  24. // args and run
  25. ClientArguments args = new ClientArguments(runArgs.toArray(new String[runArgs.size()]), sparkConf);
  26. Client client = new Client(args, config, sparkConf);
  27. ApplicationId applicationId = client.submitApplication();
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement