Guest User

Untitled

a guest
Apr 25th, 2018
92
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 4.13 KB | None | 0 0
  1. # simple_ar_etl.rb
  2. # SimpleArEtl
  3. class SimpleArEtl
  4. require 'fastercsv'
  5.  
  6. def initialize(ar_class)
  7. @ar_class = ar_class
  8. end
  9.  
  10. def load(source)
  11. @source = source
  12. @rows = FasterCSV.read(@source)
  13. read_column_names
  14. end
  15.  
  16. def save
  17. if !@normalized_column_names
  18. @normalized_column_names = []
  19. end
  20.  
  21. @rows.each do |row|
  22. #create a hash of column:value to pass to activerecord
  23. values = {}
  24. row.each_index {|i|
  25. values[@columns[i].downcase] = row[i]
  26. }
  27.  
  28. #either get an existing object, or create a new one
  29. new_obj = @ar_class.find(:first, :conditions => ["external_id = ?", values["id"]])
  30. if !new_object
  31. new_obj = @ar_class.new(values.dup.delete_if{|key, value| @normalized_column_names.include?(key) })
  32. end
  33.  
  34. # normalize field
  35. process_normalized_with_habtm(values, new_obj)
  36.  
  37. # attach this to another object
  38. if @attached_class
  39. attached_object = @attached_class.find(:first, :conditions => [@klass_pk_field.to_s + " = ?", values[@fk_field]])
  40. if attached_object
  41. attached_collection = attached_object.send(new_obj.class.table_name.pluralize)
  42. attached_collection << new_obj
  43. new_obj.save!
  44. end
  45. else
  46. # we weren't attaching this to anything, just save it
  47. new_obj.save!
  48. end
  49. end # @rows.each
  50. end
  51.  
  52. #TODO: this should be able to be called before load as well (just cache the changes to be made, and make them on import)
  53. def rename_field(old_name, new_name)
  54. @columns.map! {|item|
  55. if item.downcase == old_name.downcase
  56. new_name.downcase
  57. else
  58. item.downcase
  59. end
  60. }
  61. end
  62.  
  63. #TODO: this should be able to be called before load as well (just cache the changes to be made, and make them on import)
  64. def delete_field(field_name)
  65. index_of = @columns.index(field_name)
  66. @rows.map! {|item|
  67. item.delete_at(index_of)
  68. item
  69. }
  70. @columns.delete_at(index_of)
  71. end
  72.  
  73. #TODO: this should support multiple attached classes
  74. def attach_to(klass, klass_pk_field, fk_field)
  75. @attached_class = klass
  76. @fk_field = fk_field
  77. @klass_pk_field = klass_pk_field
  78. end
  79.  
  80. # TODO: this should support multiple normalizations eventually
  81. def normalize_with_habtm(normalized_klass, link_klass, normalized_column_names, comparison_field)
  82. @normalized_klass = normalized_klass
  83. @link_klass = link_klass
  84. @normalized_column_names = normalized_column_names
  85. @comparison_field = comparison_field
  86. end
  87.  
  88. # these methods are here only for backwards compatibility
  89. def self.rename_field(row, old_name, new_name)
  90. old_val = row.delete(old_name)
  91. if old_val
  92. row[new_name] = old_val
  93. end
  94. row
  95. end
  96.  
  97. def self.delete_field(row, field_name)
  98. row.delete(field_name)
  99. row
  100. end
  101.  
  102.  
  103. private
  104. def read_column_names
  105. # read the column names, and remove that row
  106. @columns = @rows[0]
  107. @rows.delete_at(0)
  108.  
  109. end
  110.  
  111. def process_normalized_with_habtm(values, new_obj)
  112. if(@normalized_klass && @link_klass && @comparison_field)
  113. @normalized_klass.find(:all).each do |norm|
  114. if values.has_key?(norm.send @comparison_field) && values[norm.send @comparison_field] == "1"
  115. link_object = @link_klass.new
  116. link_object.send new_obj.class.name.downcase + "=", new_obj
  117. link_object.send norm.class.name.downcase + "=", norm
  118. link_object.save!
  119. end
  120. end
  121. end
  122. end
  123.  
  124. end
  125.  
  126. # call to load/save etc
  127. importer = SimpleArEtl.new(Person)
  128. importer.load(path)
  129.  
  130. importer.rename_field("id", "external_id")
  131. importer.rename_field("av_add", "av_address")
  132. importer.rename_field("dob", "date_of_birth")
  133. importer.rename_field("ppstart", "pp_start")
  134. importer.rename_field("ppexpdate", "pp_exp_date")
  135. importer.rename_field("warrantdat", "warrant_date")
  136. importer.rename_field("warranttyp", "warrant_type")
  137. importer.rename_field("suptype", "sup_type")
  138. importer.rename_field("suptype2", "sup_type2")
  139. importer.rename_field("istatus", "i_status")
  140. importer.rename_field("lname", "last_name")
  141. importer.rename_field("fname", "first_name")
  142. importer.rename_field("resdate", "res_date")
  143. importer.rename_field("addtype", "add_type")
  144.  
  145. importer.save()
Add Comment
Please, Sign In to add comment