Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- val london_crime_records_DS = spark.read.format("org.apache.spark.csv").
- option("header", true).option("inferSchema", true).
- csv("C:\\Users\\Luk3r\\Desktop\\london-crime\\CityofLondonPoliceRecords.csv").
- cache();
- london_crime_records_DS.union(metropolitan_crime_records_DS).dropDuplicates("Crime type").withColumn("id", functions.monotonically_increasing_id).select("id","Crime type").collect().foreach(x => println(x))
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement