Guest User

Untitled

a guest
Jul 18th, 2018
112
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 1.74 KB | None | 0 0
  1. /**
  2. * Add the JAR at the given HDFS path to the Classpath
  3. *
  4. * @param conf
  5. * @param path
  6. */
  7. public static void addJarToJobClasspath(JobConf conf, String path) {
  8. addJarsToJobClasspath(conf, new String[] { path });
  9. }
  10.  
  11. /**
  12. * Add the JARs from the given HDFS paths to the Classpath
  13. *
  14. * @param conf
  15. * @param paths
  16. */
  17. public static void addJarsToJobClasspath(JobConf conf, String[] paths) {
  18.  
  19. URL[] libjars = new URL[paths.length];
  20.  
  21. // Grab JARs from HDFS and cache locally
  22. for (int i = 0; i < paths.length; i++) {
  23. try {
  24. String hdfsPath = paths[i];
  25. Path srcPath = new Path(hdfsPath);
  26.  
  27. FileSystem fs = FileSystem.get(conf);
  28. File dst = File.createTempFile(srcPath.getName() + "-", ".jar");
  29. dst.deleteOnExit();
  30. Path dstPath = new Path(dst.getAbsolutePath());
  31.  
  32. fs.copyToLocalFile(srcPath, dstPath);
  33. libjars[i] = new URL("file:" + dst.getAbsolutePath());
  34.  
  35. } catch (IOException ex) {
  36. throw new RuntimeException("Error setting up classpath", ex);
  37. }
  38. }
  39.  
  40. // Set classloader in current conf/thread
  41. conf.setClassLoader(
  42. new URLClassLoader(libjars, conf.getClassLoader()));
  43.  
  44. Thread.currentThread().setContextClassLoader(
  45. new URLClassLoader(libjars, Thread.currentThread().getContextClassLoader()));
  46.  
  47. // Append to tmpjars variable
  48. String jars = conf.get("tmpjars");
  49. if (jars == null) {
  50. jars = "";
  51. }
  52. for (int i = 0; i < libjars.length; i++) {
  53. URL url = libjars[i];
  54. if (jars.length() > 0) {
  55. jars += ",";
  56. }
  57. jars += url.toString();
  58. }
  59. conf.set("tmpjars", jars);
  60.  
  61. }
Add Comment
Please, Sign In to add comment