Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- temp_tab_name = "mytablename";
- x = spark.sql("select * from " +temp_tab_name +" limit 10");
- x = x.persist()
- x.count() #action to activate all the above steps
- x.show() #x should have been persisted in memory here, DAG evaluated, no going back to "select..." whenever referred to
- x.is_cached #True
- spark.sql("drop table "+ temp_tab_name);
- x.is_cached #Still true!!
- x.show() # Error, table not found here
- # df method
- df = spark.range(10)
- df.count() # action to materialize df object in ram
- df.unpersist() # remove df object from ram
- # temp table method
- df.createOrReplaceTempView("df_sql")
- spark.catalog.cacheTable("df_sql")
- spark.sql("select * from df_sql").count() # action to materialize temp table in ram
- spark.catalog.uncacheTable("df_sql") # remove temp table from ram
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement