Advertisement
Guest User

ES Embedded Hadoop Writer

a guest
Oct 2nd, 2014
289
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Java 1.21 KB | None | 0 0
  1.     public static class TestReducer implements Reducer<Text, Text, LongWritable, MapWritable> {
  2.         private Node node;
  3.         private Client client;
  4.         public void configure(JobConf job) {
  5.             node = nodeBuilder().client(false).clusterName("hadoopTest").node();
  6.             client = node.client();
  7.             Map<String, Object> settings = new HashMap<>();
  8.             settings.put("location", "/tmp/es_hadoop_backups/my_backup");
  9.             settings.put("compress", true);
  10.  
  11.             PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("my_backup")
  12.                     .setType("fs").setSettings(settings).get();
  13.             boolean success = putRepositoryResponse.isAcknowledged();
  14.             client = node.client();
  15.         }
  16.  
  17.         public void reduce(Text segmentId, Iterator<Text> segmentRestObjects, OutputCollector<LongWritable, MapWritable> output, Reporter reporter) throws IOException {
  18.             String json = ".....";
  19.             client.prepareIndex("drewmarch4", "conversation").setSource(json).execute().actionGet();
  20.         }
  21.  
  22.         public void close() throws IOException {
  23.             CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("my_backup", "test-snap6").setWaitForCompletion(true).setIndices("drew*").get();
  24.             node.close();
  25.         }
  26.     }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement