Advertisement
Guest User

Untitled

a guest
Sep 26th, 2017
60
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Scala 0.35 KB | None | 0 0
  1. def time[A](f: ⇒ A) = {
  2.     val s = System.nanoTime
  3.     val ret = f
  4.     println("time: " + (System.nanoTime - s) / 1e9 + " seconds")
  5.     ret
  6. }
  7.  
  8. import org.apache.spark.sql.DataFrame
  9. def materialize(df: DataFrame, tempStorage: String = "hdfs:///spark/tmp") = {
  10.    df.write.mode("overwrite").parquet(tempStorage)
  11.    spark.read.parquet(tempStorage)
  12. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement