Guest User

Flink Kafka Producer - processElements NullPointerException

a guest
Jul 14th, 2017
185
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Java 2.13 KB | None | 0 0
  1. import org.apache.flink.streaming.api.datastream.DataStream;
  2. import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
  3. import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer010;
  4. import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
  5. import org.apache.flink.streaming.util.serialization.SerializationSchema;
  6.  
  7. import java.math.BigInteger;
  8. import java.util.*;
  9.  
  10. public class ProcessElementsFails {
  11.  
  12.     public static void main(String[] args) throws Exception {
  13.  
  14.         // setup
  15.         final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  16.         setUpKafkaProducer();
  17.  
  18.         // create input stream of List<BigInteger>
  19.         List<BigInteger> inputData = new ArrayList<>();
  20.         inputData.add(BigInteger.valueOf(1)); inputData.add(BigInteger.valueOf(2));
  21.             inputData.add(BigInteger.valueOf(3)); inputData.add(BigInteger.valueOf(-4));
  22.         DataStream<List<BigInteger>> input = env.fromElements(inputData);
  23.  
  24.         // for each value in list, if condition met, emit to Kafka
  25.         input.map(integerList -> {
  26.             for (BigInteger integer: integerList) {
  27.                 if (integer.compareTo(BigInteger.ZERO) > 0)
  28.                     emitToKafka(integer);
  29.             }
  30.             return true;}
  31.             ).returns(Boolean.class);  // (return value meaningless)
  32.  
  33.         env.execute();
  34.     }
  35.  
  36.     private static FlinkKafkaProducer010<BigInteger> kafkaProducer;
  37.  
  38.     private static void setUpKafkaProducer() {
  39.         Properties kafkaProducerProperties = new Properties();
  40.         kafkaProducerProperties.setProperty("bootstrap.servers", "localhost:9092");
  41.         kafkaProducer = new FlinkKafkaProducer010<>("process-element-tests", new IntegerSerializer(),
  42.                 kafkaProducerProperties);
  43.     }
  44.  
  45.     private static class IntegerSerializer implements SerializationSchema<BigInteger> {
  46.         @Override
  47.         public byte[] serialize(BigInteger integer) {
  48.             return integer.toByteArray();
  49.         }
  50.     }
  51.  
  52.     private static void emitToKafka(BigInteger value) throws Exception {
  53.         try {
  54.             StreamRecord<BigInteger> record = new StreamRecord<>(value);
  55.             kafkaProducer.processElement(record);
  56.             System.out.println("Element processed.");
  57.         } catch (Exception e) {
  58.             e.printStackTrace();
  59.         }
  60.     }
  61. }
Advertisement
Add Comment
Please, Sign In to add comment