Advertisement
Guest User

Untitled

a guest
Jun 10th, 2016
125
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 1.52 KB | None | 0 0
  1. import org.apache.spark.SparkConf
  2.  
  3. import java.util.Properties
  4.  
  5. import org.apache.spark.sql.SQLContext
  6.  
  7. import org.apache.spark.SparkContext
  8.  
  9. import org.apache.spark.SparkContext._
  10.  
  11. import org.apache.spark.sql.SaveMode
  12.  
  13. import java.util.Date
  14.  
  15. import java.text.SimpleDateFormat
  16.  
  17.  
  18.  
  19. // Some Class
  20.  
  21. case class Person(age: Int, name: String, occu: String)
  22.  
  23.  
  24. object Utilities {
  25.  
  26. val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
  27.  
  28. val url = "jdbc:mysql://sna-blr-02.beds.boeing.com:3306/ahmdb";
  29.  
  30. val prop = new Properties()
  31.  
  32. prop.put("user", "hadoop")
  33.  
  34. prop.put("password", "")
  35.  
  36. prop.put("driver", "com.mysql.jdbc.Driver")
  37.  
  38. val tablename = "person"
  39.  
  40.  
  41. def savePerson(person: Person, sc: SparkContext, sqlContext: SQLContext): Unit = {
  42.  
  43.  
  44. import sqlContext.implicits._
  45.  
  46.  
  47. // Get the current time
  48.  
  49. val currentTime = sdf.format(new Date());
  50.  
  51.  
  52. // Create a sample Dataframe
  53.  
  54. val df = sc.parallelize(Array((person.age, person.name, person.occu, currentTime))).toDF("age", "name", "occu", "updatedtime")
  55.  
  56.  
  57. // JDBC insert in append mode
  58.  
  59. df.write.mode(SaveMode.Append).jdbc(url, tablename , prop)
  60.  
  61. }
  62.  
  63. }
  64.  
  65.  
  66. //Above method usage:
  67.  
  68. Utilities.savePerson(33, "Foo", "bar", sparkContext, sqlContext)
  69.  
  70.  
  71. // MySql Table Structure
  72.  
  73. CREATE TABLE person(
  74.  
  75. age int(3) DEFAULT NULL,
  76.  
  77. name text NOT NULL,
  78.  
  79. occu text,
  80.  
  81. updatedtime datetime NOT NULL
  82.  
  83. ) ENGINE=InnoDB DEFAULT CHARSET=latin1;
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement