Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- val execQuery= "select header_id,source_system_name,line_num,cast(extract(epoch from last_update_date)*10000 as BIGINT) epochVals from schema.tablename where source_system_name='SAP' and period_year=2018 and period_num=3"
- val yearDF = spark.read.format("jdbc").option("url", connectionUrl)
- .option("dbtable", s"(${execQuery}) as year2016")
- .option("user", devUserName)
- .option("password", devPassword)
- .option("partitionColumn","epochVals")
- .option("lowerBound", minEp)
- .option("upperBound", maxEp)
- .option("numPartitions",20)
- .load()
- println("Number of partitions: " + yearDF.rdd.getNumPartitions) -> Number of partitions: 1
- val yearDF = spark.read.format("jdbc").option("url", connectionUrl)
- .option("dbtable", s"(${execQuery}) as year2016")
- .option("user", devUserName)
- .option("password", devPassword)
- .option("partitionColumn","header_id")
- .option("lowerBound", minEp)
- .option("upperBound", maxEp)
- .option("numPartitions",20)
- .load()
- println("Number of partitions: " + yearDF.rdd.getNumPartitions) -> Number of partitions: 20
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement