Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import org.apache.spark.sql.functions._
- import org.apache.spark.sql._
- import org.apache.spark.sql.types._
- val spark: SparkSession = SparkSession.builder().
- appName("myapp").master("local").getOrCreate()
- case class Person(id: Int, date: String)
- import spark.implicits._
- val mydf: DataFrame = Seq(Person(1,"9/16/13")).toDF()
- val select_df: DataFrame = mydf.select(unix_timestamp(mydf("date"),"MM/dd/yy").cast(TimestampType))
- select_df.select(year($"date")).show()
- import org.apache.spark.sql.functions._
- import org.apache.spark.sql.types._
- val df=Seq(("2019-07-19")).toDF("date")
- //all the below actions results same i.e. year extracting from date column.
- df.select(year('date).alias("new_date")).show()
- df.select(year(df("date")).alias("new_date")).show()
- df.select(year($"date").alias("new_date")).show()
- df.withColumn("new_date",year('date)).select("new_date").show()
- +--------+
- |new_date|
- +--------+
- | 2019|
- +--------+
- case class Person(id: Int, date: String)
- val mydf = Seq(Person(1,"9/16/13")).toDF
- val solution = mydf.withColumn("year", year(to_timestamp($"date", "MM/dd/yy")))
- scala> solution.show
- +---+-------+----+
- | id| date|year|
- +---+-------+----+
- | 1|9/16/13|2013|
- +---+-------+----+
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement