Advertisement
aironman

HDFS

Dec 10th, 2014
207
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. public class MyHadoopUtils {
  2.  
  3. final static Logger LOGGER = Logger.getLogger(MyHadoopUtils.class);
  4. private static FileSystem fs;
  5.  
  6. static{
  7. boolean initiated=false;
  8. Configuration myConf = new Configuration();
  9. //this is namenode url
  10. myConf.set("fs.defaultFS", CalculateLevelFiles.getDEFAULT_FS());
  11. LOGGER.info("Started using " + CalculateLevelFiles.getDEFAULT_FS());
  12. try {
  13. fs = FileSystem.get(myConf);
  14. initiated=true;
  15. } catch (IOException e) {
  16. LOGGER.fatal("ATTENTION! an error occurred when trying to instantiate HDFS handler!" + e.getLocalizedMessage());
  17. e.printStackTrace();
  18. }finally{
  19. LOGGER.debug("Completed. initiated? " + initiated);
  20. }
  21. }
  22.  
  23. /***
  24. * a method that writes data provided by the StringBuffer to a path.
  25. * The path must exists
  26. * @param path the absolute path within HDFS
  27. * @param sb the buffer with data
  28. * @return true if ok, false if not
  29. */
  30. public static boolean saveWithinHDFS(String path,StringBuffer sb){
  31.  
  32. LOGGER.debug("Started with path: " + path);
  33. boolean isOk=false;
  34. Path pt = new Path(path);
  35. try {
  36. BufferedWriter br = new BufferedWriter(new OutputStreamWriter(fs.create(pt, true)));
  37. br.write(sb.toString());
  38. br.close();
  39. isOk=true;
  40. String [] adata = sb.toString().split("\n");
  41. //LOGGER.info("ATTENTION A file has been written to HDFS! path: " + path);
  42. LOGGER.info(adata!=null && adata.length > 0?adata.length:"Zero " + " records written to " + path);
  43. } catch (IOException e) {
  44. LOGGER.warn("ATTENTION! it is been imposible to save a file within HDFS!. path: " + path);
  45. e.printStackTrace();
  46. } finally{
  47. LOGGER.debug("Completed isOk: " + isOk);
  48. }
  49.  
  50. return isOk;
  51. }
Advertisement
Advertisement
Advertisement
RAW Paste Data Copied
Advertisement