Advertisement
Guest User

Untitled

a guest
Dec 12th, 2016
110
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 2.45 KB | None | 0 0
  1. package test
  2. import org.apache.spark.SparkContext
  3. import org.apache.spark.SparkContext._
  4. import org.apache.spark.SparkConf
  5. import java.sql.DriverManager
  6. import java.sql.Connection
  7. import java.sql.SQLException;
  8.  
  9. object Main {
  10. def main(args: Array[String]) {
  11. if(args.length<3){
  12. println("Not enough args")
  13. System.exit(1)
  14. }
  15. val driver = "com.mysql.jdbc.Driver"
  16. val url = "jdbc:mysql://172.20.140.245/db_stat"
  17. val username = "db_stat"
  18. val password = "pass4db!"
  19. var connection:Connection = null
  20. //val conf = new SparkConf().setAppName("WordCounter").setMaster("yarn-client")
  21. //val sc = new SparkContext(conf)
  22. //val text_file = sc.textFile(args(0))
  23. //val text_names = sc.textFile(args(2))
  24. //val names = text_names.flatMap(line => line.split(",") ).collect()
  25. //val lines = text_file.flatMap( line => line.split(" ") )
  26. //val counts = lines.filter(line => names.contains(line)).map(word => (word, 1)).reduceByKey(_ + _)
  27. //val summary = counts.collect()
  28. //counts.saveAsTextFile(args(1))
  29. try {
  30. // make the connection
  31. Class.forName(driver)
  32. connection = DriverManager.getConnection(url, username, password)
  33.  
  34. // create the statement, and run the select query
  35. val statement = connection.createStatement()
  36. val sql = "SELECT * FROM DIM_key_brand where keybrand_name_status = 'Y';"
  37. val resultSet = statement.executeQuery(sql)
  38. val mapWithBrands = scala.collection.mutable.Map[String,Int]()
  39. while ( resultSet.next() ) {
  40. val keybrand_name = resultSet.getString("keybrand_name")
  41. val keybrand_id = resultSet.getInt("keybrand_id")
  42. mapWithBrands(keybrand_name) = keybrand_id
  43. println("brand = " + keybrand_name + " " + keybrand_id)
  44. }
  45. val names = mapWithBrands.keys
  46. //TODO: insert code
  47.  
  48.  
  49.  
  50. // create the statement, and run the select query
  51. val sqlInsert = "INSERT INTO FACT_brand_mentioned VALUES(?,?)"
  52.  
  53. val statementInsert = connection.prepareStatement(sqlInsert)
  54. def ins(a:(String, Int)){
  55. statementInsert.setInt(1, a._2 )
  56. statementInsert.setInt(2,mapWithBrands(a._1) )
  57. val resultSet = statementInsert.executeUpdate()
  58. }
  59. //summary.foreach { x=>ins(x) }
  60. } catch {
  61. case ex => println("SQLException: " + ex.getMessage())
  62.  
  63. }
  64. connection.close()
  65. }
  66.  
  67. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement