Guest User

Untitled

a guest
Apr 17th, 2018
134
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 1.97 KB | None | 0 0
  1. /**
  2. * This checks that the two columns are compatible. Here, compatible means that:
  3. * - Their names are resolvable (usually only case differences)
  4. * - AND
  5. * - They are either:
  6. * -- The same dataType
  7. * -- StructTypes, for which common sub fields are all (recursively) compatible.
  8. *
  9. * @param ours our schema
  10. * @param theirs their schema
  11. * @return
  12. */
  13. private def columnsCompatible(
  14. ours: StructField,
  15. theirs: StructField,
  16. resolver: Resolver
  17. ): Boolean = {
  18. // If we can't resolve between the column names, then no these could be compatible.
  19. if (!resolver(ours.name, theirs.name)) {
  20. false
  21. }
  22.  
  23. // Else ok, we resolved names and they have same dataType, cool!
  24. else if (ours.dataType == theirs.dataType) {
  25. true
  26. }
  27.  
  28. // Else if we get here, the types didn't match, but both types were StructTypes.
  29. // We need to recurse into theirs to check each sub field.
  30. else if (isStructType(ours) && isStructType(theirs)) {
  31. // Check that any fields in theirSchema that are also in our schema
  32. // have identical types. The exception is StructType fields, for which
  33. // we will recurse and ensure that each of the contained StructFields
  34. // themselves have identical types.
  35. val ourSchema = ours.dataType.asInstanceOf[StructType]
  36. val theirSchema = theirs.dataType.asInstanceOf[StructType]
  37.  
  38. // Get all fields in theirSchema that are also in ourSchema
  39. // and make sure that all of these fields are also compatible,
  40. // Recursing if needed.
  41. theirSchema
  42. .filter(field => findColumnByName(ourSchema, field.name, resolver).isDefined)
  43. .forall { theirField => columnsCompatible(
  44. theirField,
  45. findColumnByName(ourSchema, theirField.name, resolver).get,
  46. resolver
  47. )}
  48.  
  49. }
  50.  
  51. // Else at least one of the dataTypes was not a StructType,
  52. // definitely not compatible.
  53. else {
  54. false
  55. }
  56. }
Add Comment
Please, Sign In to add comment