Guest User

Untitled

a guest
Jun 13th, 2018
73
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 1.35 KB | None | 0 0
  1. # +---+-----+-----+
  2. # | id|d_var|d_val|
  3. # +---+-----+-----+
  4. # |a01| 112| null|
  5. # |a01| 113| 0|
  6. # |a02| 112| null|
  7. # |a02| 113| 0|
  8. # +---+-----+-----+
  9.  
  10. dfA = spark.createDataFrame(
  11. [
  12. ('a01', '112', None),
  13. ('a01', '113', '0'),
  14. ('a02', '112', None),
  15. ('a02', '113', '0')
  16. ],
  17. ('id', 'd_var', 'd_val')
  18. )
  19.  
  20. # +---+-----+-----+------+-----+
  21. # | id|d_var|d_val|c_type|c_val|
  22. # +---+-----+-----+------+-----+
  23. # |a01| 112| null| red| 1|
  24. # |a01| 113| 0| red| 1|
  25. # +---+-----+-----+------+-----+
  26.  
  27. dfB = spark.createDataFrame(
  28. [
  29. ('a01', '112', None, 'red', '1'),
  30. ('a01', '113', '0', 'red', '1')
  31. ],
  32. ('id', 'd_var', 'd_val', 'c_type', 'c_val')
  33. )
  34.  
  35. static_cols = dfB.columns[:3]
  36. dfA.join(dfB, static_cols, how='left').orderBy('id', 'd_var').show()
  37.  
  38. # +---+-----+-----+------+-----+
  39. # | id|d_var|d_val|c_type|c_val|
  40. # +---+-----+-----+------+-----+
  41. # |a01| 112| null| null| null| <-
  42. # |a01| 113| 0| red| 1|
  43. # |a02| 112| null| null| null|
  44. # |a02| 113| 0| null| null|
  45. # +---+-----+-----+------+-----+
  46.  
  47. # +---+-----+-----+------+-----+
  48. # | id|d_var|d_val|c_type|c_val|
  49. # +---+-----+-----+------+-----+
  50. # |a01| 112| null| red| 1| <-
  51. # |a01| 113| 0| red| 1|
  52. # |a02| 112| null| null| null|
  53. # |a02| 113| 0| null| null|
  54. # +---+-----+-----+------+-----+
Add Comment
Please, Sign In to add comment