Advertisement
Guest User

Untitled

a guest
Mar 19th, 2019
48
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 1.88 KB | None | 0 0
  1. //I get an Object from Hbase here
  2. val objectRDD : RDD[HbaseRecord] = ...
  3.  
  4. //I convert the RDD[HbaseRecord] into RDD[Row]
  5. val rowRDD : RDD[Row] = objectRDD.map(
  6. hbaseRecord => {
  7.  
  8. // Simple types
  9.  
  10. val uuid : String = hbaseRecord.uuid
  11. val timestamp : String = hbaseRecord.timestamp
  12.  
  13.  
  14. // Maps
  15.  
  16. val name = Row(hbaseRecord.nameMap.firstName.getOrElse(""),
  17. hbaseRecord.nameMap.middleName.getOrElse(""),
  18. hbaseRecord.nameMap.lastName.getOrElse(""))
  19.  
  20.  
  21. // Parsing maps of maps
  22.  
  23. val contactsMap = hbaseRecord.contactsMap
  24.  
  25. val homeContactMap = contactsMap.get("HOME")
  26. val homeContact = Row(homeContactMap.contactType,
  27. homeContactMap.areaCode,
  28. homeContactMap.number)
  29.  
  30. val workContactMap = contactsMap.get("WORK")
  31. val workContact = Row(workContactMap.contactType,
  32. workContactMap.areaCode,
  33. workContactMap.number)
  34.  
  35. val contacts = Row(homeContact,workContact)
  36.  
  37. Row(uuid, timestamp, name, contacts)
  38.  
  39. }
  40. )
  41.  
  42.  
  43. //Here I define the schema
  44. val schema = new StructType()
  45. .add("uuid",StringType)
  46. .add("timestamp", StringType)
  47. .add("name", new StructType()
  48. .add("firstName",StringType)
  49. .add("middleName",StringType)
  50. .add("lastName",StringType)
  51. .add("contacts", new StructType(
  52. Array(
  53. StructField("contactType", StringType),
  54. StructField("areaCode", StringType),
  55. StructField("number", StringType)
  56. )))
  57.  
  58.  
  59. //Now I try to create a Dataframe using the RDD[Row] and the schema
  60. val dataFrame = sqlContext.createDataFrame(rowRDD , schema)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement