Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- /* Database Configuration*/
- val jdbcURL = s"jdbc:mysql://${HOST}/${DATABASE}"
- val jdbcUsername = s"${USERNAME}"
- val jdbcPassword = s"${PASSWORD}"
- import java.util.Properties
- val connectionProperties = new Properties()
- connectionProperties.put("user", jdbcUsername)
- connectionProperties.put("password", jdbcPassword)
- connectionProperties.put("driver", "com.mysql.cj.jdbc.Driver")
- /* Read Data from MySQL */
- val desiredData = spark.read.jdbc(jdbcURL, "${TABLE NAME}", connectionProperties)
- desiredData.printSchema
- /* Data Manipulation */
- desiredData.createOrReplaceTempView("desiredData")
- val query = s"""
- SELECT COUNT(*) AS `Record Number`
- FROM desiredData
- """
- spark.sql(query).show
- val query2 = s"""
- SELECT ROW_NUMBER() OVER (PARTITION BY column1 ORDER BY column1, column2) AS column3
- FROM desiredData
- """
- spark.sql(query2).show
- .
- .
- .
Add Comment
Please, Sign In to add comment