Advertisement
Guest User

Untitled

a guest
Aug 19th, 2019
125
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Scala 0.83 KB | None | 0 0
  1.  def main(args: Array[String]): Unit = {
  2.     printf("Infering schema from %s\n", wiki_pages)
  3.     val (sc, conf) = Util.createSpark();
  4.     {
  5.       // from https://github.com/Teradata/kylo/raw/master/samples/sample-data/parquet/userdata1.parquet
  6.       val df = sc.read.parquet("swift://wiki.OVH/userdata1.parquet")
  7.       printf("There are %d items\n", df.count())
  8.       df.show()
  9.       //df.repartition(5).write.mode("overwrite").parquet("swift://wiki.OVH/userdata1-rewrite.parquet")
  10.       //df.write.mode("overwrite").parquet("swift://wiki.OVH/userdata1-rewrite.parquet")
  11.       df.write.mode("overwrite").format("parquet").save("swift://wiki.OVH/userdata1-rewrite.parquet")
  12.     }
  13.     {
  14.       val df = sc.read.parquet("swift://wiki.OVH/userdata1-rewrite.parquet")
  15.       printf("There are %d items\n", df.count())
  16.       df.show()
  17.     }
  18. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement