Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- def main(args: Array[String]): Unit = {
- printf("Infering schema from %s\n", wiki_pages)
- val (sc, conf) = Util.createSpark();
- {
- // from https://github.com/Teradata/kylo/raw/master/samples/sample-data/parquet/userdata1.parquet
- val df = sc.read.parquet("swift://wiki.OVH/userdata1.parquet")
- printf("There are %d items\n", df.count())
- df.show()
- //df.repartition(5).write.mode("overwrite").parquet("swift://wiki.OVH/userdata1-rewrite.parquet")
- //df.write.mode("overwrite").parquet("swift://wiki.OVH/userdata1-rewrite.parquet")
- df.write.mode("overwrite").format("parquet").save("swift://wiki.OVH/userdata1-rewrite.parquet")
- }
- {
- val df = sc.read.parquet("swift://wiki.OVH/userdata1-rewrite.parquet")
- printf("There are %d items\n", df.count())
- df.show()
- }
- }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement