Guest User

Untitled

a guest
Jul 23rd, 2018
78
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 1.19 KB | None | 0 0
  1. JavaRDD<Tuple2<List<Row>,SourceParam>> scanRDDs = context.parallelize(Collections.emptyList());
  2. for(DefinitionSource eachSource : definition.getSources()){
  3. if(!source.isExcludeDelta()){
  4. scanRDDs.union(context.parallelize(Collections.singletonList(new Tuple2<>(
  5. lookupService.scan(context,transUtils.tableName(eachSource.getSchemaName(),
  6. eachSource.getTableName()),param.getTimeSlot(),eachSource.getTimelineKey()).collect(),
  7. new SourceParam(eachSource,param)))));
  8. }
  9. }
  10.  
  11. // Instead of
  12.  
  13. // List<Tuple2<List<Row>, SourceParam>> scanRDDs = new ArrayList<>();
  14. // Definition definition = definitions.getDefinition(param.getName());
  15. // scanRDDs.addAll(definition.getSources().parallelStream()
  16. // .filter(source -> !source.isExcludeDelta())
  17. // .map(source -> new Tuple2<>(lookupService.scan(context,
  18. // transUtils.tableName(source.getSchemaName(),
  19. // source.getTableName()), param.getTimeSlot(),
  20. // source.getTimelineKey()).collect(), new SourceParam(source, param)))
  21. // .filter(source -> !source._1.isEmpty())
  22. // .collect(Collectors.toList()));
  23.  
  24. // JavaPairRDD<String, CSVRow> finalRdds = context
  25. // .parallelize(scanRDDs, scanRDDs.size())... and so on
Add Comment
Please, Sign In to add comment