Advertisement
Guest User

Untitled

a guest
Feb 21st, 2019
75
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 1.63 KB | None | 0 0
  1. val execQuery= "select header_id,source_system_name,line_num,cast(extract(epoch from last_update_date)*10000 as BIGINT) epochVals from schema.tablename where source_system_name='SAP' and period_year=2018 and period_num=3"
  2. val yearDF = spark.read.format("jdbc").option("url", connectionUrl)
  3. .option("dbtable", s"(${execQuery}) as year2016")
  4. .option("user", devUserName)
  5. .option("password", devPassword)
  6. .option("partitionColumn","epochVals")
  7. .option("lowerBound", minEp)
  8. .option("upperBound", maxEp)
  9. .option("numPartitions",20)
  10. .load()
  11.  
  12. println("Number of partitions: " + yearDF.rdd.getNumPartitions) -> Number of partitions: 1
  13.  
  14. val yearDF = spark.read.format("jdbc").option("url", connectionUrl)
  15. .option("dbtable", s"(${execQuery}) as year2016")
  16. .option("user", devUserName)
  17. .option("password", devPassword)
  18. .option("partitionColumn","header_id")
  19. .option("lowerBound", minEp)
  20. .option("upperBound", maxEp)
  21. .option("numPartitions",20)
  22. .load()
  23.  
  24. println("Number of partitions: " + yearDF.rdd.getNumPartitions) -> Number of partitions: 20
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement