Advertisement
Guest User

Untitled

a guest
Jul 20th, 2019
63
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 1.22 KB | None | 0 0
  1. import org.apache.spark.sql.functions._
  2. import org.apache.spark.sql._
  3. import org.apache.spark.sql.types._
  4. val spark: SparkSession = SparkSession.builder().
  5. appName("myapp").master("local").getOrCreate()
  6.  
  7. case class Person(id: Int, date: String)
  8. import spark.implicits._
  9.  
  10. val mydf: DataFrame = Seq(Person(1,"9/16/13")).toDF()
  11. val select_df: DataFrame = mydf.select(unix_timestamp(mydf("date"),"MM/dd/yy").cast(TimestampType))
  12. select_df.select(year($"date")).show()
  13.  
  14. import org.apache.spark.sql.functions._
  15. import org.apache.spark.sql.types._
  16. val df=Seq(("2019-07-19")).toDF("date")
  17.  
  18. //all the below actions results same i.e. year extracting from date column.
  19. df.select(year('date).alias("new_date")).show()
  20. df.select(year(df("date")).alias("new_date")).show()
  21. df.select(year($"date").alias("new_date")).show()
  22. df.withColumn("new_date",year('date)).select("new_date").show()
  23.  
  24. +--------+
  25. |new_date|
  26. +--------+
  27. | 2019|
  28. +--------+
  29.  
  30. case class Person(id: Int, date: String)
  31. val mydf = Seq(Person(1,"9/16/13")).toDF
  32. val solution = mydf.withColumn("year", year(to_timestamp($"date", "MM/dd/yy")))
  33. scala> solution.show
  34. +---+-------+----+
  35. | id| date|year|
  36. +---+-------+----+
  37. | 1|9/16/13|2013|
  38. +---+-------+----+
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement