Guest User

Untitled

a guest
Jan 19th, 2018
74
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 1.85 KB | None | 0 0
  1. FileSystem fs = FileSystem.get(conf);
  2. FSDataOutputStream FS = fs.create(new Path("S3 folder output path"+"//Result.txt"));
  3.  
  4. PrintWriter writer = new PrintWriter(FS);
  5. writer.write(averageDelay.toString());
  6. writer.close();
  7. FS.close();
  8.  
  9. FileSystem fileSystem = FileSystem.get(URI.create(otherArgs[1]),conf);
  10. FSDataOutputStream fsDataOutputStream = fileSystem.create(new Path(otherArgs[1]+"//Result.txt"));
  11. PrintWriter writer = new PrintWriter(fsDataOutputStream);
  12. writer.write("n Average Delay:"+averageDelay);
  13. writer.close();
  14. fsDataOutputStream.close();
  15.  
  16. FileSystem fileSystem = FileSystem.get(URI.create(otherArgs[1]),new JobConf(<Your_Class_Name_here>.class));
  17. FSDataOutputStream fsDataOutputStream = fileSystem.create(new
  18. Path(otherArgs[1]+"//Result.txt"));
  19. PrintWriter writer = new PrintWriter(fsDataOutputStream);
  20. writer.write("n Average Delay:"+averageDelay);
  21. writer.close();
  22. fsDataOutputStream.close();
  23.  
  24. ... AmazonS3 s3Client;
  25.  
  26. public void reduce(Text key, java.lang.Iterable<Text> values, Reducer<Text, Text, Text, Text>.Context context) throws Exception {
  27.  
  28. UUID fileUUID = UUID.randomUUID();
  29. SimpleDateFormat sdf = new SimpleDateFormat("yyy-MM-dd");
  30. sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
  31.  
  32. String fileName = String.format("nightly-dump/%s/%s-%s",sdf.format(new Date()), key, fileUUID);
  33. log.info("Filename = [{}]", fileName);
  34.  
  35. String content = "";
  36. int count = 0;
  37. for (Text value : values) {
  38. count++;
  39. String s3Line = value.toString();
  40. content += s3Line + "n";
  41. }
  42. log.info("Count = {}, S3Lines = n{}", count, content);
  43.  
  44.  
  45. PutObjectResult putObjectResult = s3Client.putObject(S3_BUCKETNAME, fileName, content);
  46. log.info("Put versionId = {}", putObjectResult.getVersionId());
  47.  
  48. reduceWriteContext("1", "1");
  49.  
  50. context.setStatus("COMPLETED");
  51. }
Add Comment
Please, Sign In to add comment