Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- def time[A](f: ⇒ A) = {
- val s = System.nanoTime
- val ret = f
- println("time: " + (System.nanoTime - s) / 1e9 + " seconds")
- ret
- }
- import org.apache.spark.sql.DataFrame
- def materialize(df: DataFrame, tempStorage: String = "hdfs:///spark/tmp") = {
- df.write.mode("overwrite").parquet(tempStorage)
- spark.read.parquet(tempStorage)
- }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement