Guest User

Untitled

a guest
Jan 16th, 2019
100
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 1.43 KB | None | 0 0
  1. {"identifier": "xxx", "value": 10.0, "ts":"2019-01-16T10:51:26.326242+0000"}
  2.  
  3. import org.apache.beam.sdk.io.kafka.KafkaIO;
  4. pipeline.apply(KafkaIO.<Long, String>read()
  5. .withBootstrapServers("kafka:9092")
  6. .withTopic("test")
  7. .withKeyDeserializer(LongDeserializer.class)
  8. .withValueDeserializer(StringDeserializer.class)
  9. .updateConsumerProperties(ImmutableMap.of("enable.auto.commit", "true"))
  10. .updateConsumerProperties(ImmutableMap.of("group.id", "Consumer1"))
  11. .commitOffsetsInFinalize()
  12. .withoutMetadata()))
  13.  
  14. PCollection<Temperature> tempCollection = p.apply(new SetupKafka())
  15. .apply(ParDo.of(new ReadFromTopic()))
  16. .apply("ParseTemperature", ParDo.of(new ParseTemperature()));
  17.  
  18. tempCollection.apply("AssignTimeStamps", WithTimestamps.of(us -> new Instant(us.getTimestamp())));
  19.  
  20. PCollection<Output> output = tempCollection.apply(Window
  21. .<Temperature>into(FixedWindows.of(Duration.standardSeconds(30)))
  22. .triggering(AfterWatermark.pastEndOfWindow()
  23. .withLateFirings(AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.standardSeconds(10))))
  24. .withAllowedLateness(Duration.standardDays(1))
  25. .accumulatingFiredPanes())
  26. .apply(new ComputeMax());
Add Comment
Please, Sign In to add comment