Advertisement
Guest User

Untitled

a guest
Jan 10th, 2017
94
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 0.98 KB | None | 0 0
  1. %pyspark
  2.  
  3. from pyspark.sql import SQLContext, Row
  4. import sys
  5. from pyspark.sql.window import Window
  6. import pyspark.sql.functions as func
  7.  
  8. #Load the data
  9. aquery = "(SELECT serial_number, min(date_time) min_date_time from schema.table where serial_number in ('abcdefg','1234567') group by serial_number) as minDates"
  10.  
  11. dfMinDates = sqlContext.read.format('jdbc').options(url='jdbc:postgresql://dadadadaaaredshift.amazonaws.com:5439/idw?tcpKeepAlive=true&ssl=true&sslfactory=org.postgresql.ssl.NonValidatingFactory?user=user&password=password', dbtable=aquery).load()
  12. dfMinDates.show()
  13.  
  14. %pyspark
  15.  
  16. aquery = "(SELECT serial_number, min(date_time) min_date_time from schema.table where serial_number in ('abcdefg','1234567') group by serial_number) as minDates"
  17.  
  18. dfMinDates = spark.read.jdbc("jdbc:postgresql://dadadadaaaredshift.amazonaws.com:5439/idw?tcpKeepAlive=true&ssl=true&sslfactory=org.postgresql.ssl.NonValidatingFactory?user=user&password=password", dbtable=aquery).load()
  19. dfMinDates.show()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement