Guest User

Untitled

a guest
Jan 19th, 2018
51
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 5.82 KB | None | 0 0
  1. ActiveRecord::Base.logger = Logger.new(STDOUT)
  2.  
  3. UNIQUE_INDEXES_FOR_MODELS = {
  4. # Just having :ems_id & :ems_ref
  5. ContainerBuild => [:ems_id, :ems_ref],
  6. ContainerBuildPod => [:ems_id, :ems_ref],
  7. ContainerGroup => [:ems_id, :ems_ref],
  8. ContainerLimit => [:ems_id, :ems_ref],
  9. ContainerNode => [:ems_id, :ems_ref],
  10. ContainerProject => [:ems_id, :ems_ref],
  11. ContainerQuota => [:ems_id, :ems_ref],
  12. ContainerReplicator => [:ems_id, :ems_ref],
  13. ContainerRoute => [:ems_id, :ems_ref],
  14. ContainerService => [:ems_id, :ems_ref],
  15. ContainerTemplate => [:ems_id, :ems_ref],
  16. Container => [:ems_id, :ems_ref],
  17. PersistentVolumeClaim => [:ems_id, :ems_ref],
  18. # Having :ems_id but not ems_ref
  19. ContainerImage => [:ems_id, :image_ref],
  20. ContainerImageRegistry => [:ems_id, :host, :port],
  21. # Nested tables, not having :ems_id and the foreign_key is a part of the unique index
  22. ContainerCondition => [:container_entity_id, :container_entity_type, :name],
  23. SecurityContext => [:resource_id, :resource_type],
  24. ComputerSystem => [:managed_entity_id, :managed_entity_type],
  25. ContainerEnvVar => [:container_id, :name, :value, :field_path],
  26. ContainerLimitItem => [:container_limit_id, :resource, :item_type],
  27. ContainerPortConfig => [:container_id, :ems_ref],
  28. ContainerQuotaScope => [:container_quota_id, :scope],
  29. ContainerQuotaItem => [:container_quota_id, :resource],
  30. ContainerServicePortConfig => [:container_service_id, :name],
  31. ContainerTemplateParameter => [:container_template_id, :name],
  32. ContainerVolume => [:parent_id, :parent_type, :name],
  33. CustomAttribute => [:resource_id, :resource_type, :name, :unique_name, :section, :source],
  34. Hardware => [:vm_or_template_id, :host_id, :computer_system_id],
  35. OperatingSystem => [:vm_or_template_id, :host_id, :computer_system_id],
  36. }.freeze
  37.  
  38. NUMBER_OF_RECORDS = 1_200_000
  39. NUMBER_OF_DUPLICATES = 3
  40. BATCH_SIZE = 10_000
  41.  
  42. class MockSaver
  43. include ManagerRefresh::SaveCollection::Saver::SqlHelper
  44.  
  45. def initialize(model_class, unique_index_columns)
  46. @model_class = model_class
  47. @table_name = model_class.table_name
  48. @unique_index_columns = unique_index_columns
  49. end
  50.  
  51. def build_data(model, foreign_key_value, string_value)
  52. data = model_unique_keys(model).each_with_object({}) do |key, obj|
  53. obj[key] = build_value(key, foreign_key_value, string_value)
  54. end
  55.  
  56. assign_attributes_for_create!(data)
  57.  
  58. data
  59. end
  60.  
  61. private
  62.  
  63. attr_reader :unique_index_columns, :table_name, :model_class
  64.  
  65. def time_now
  66. @time_now ||= Time.now.utc
  67. end
  68.  
  69. def primary_key
  70. :id
  71. end
  72.  
  73. def build_value(key, foreign_key_value, string_value)
  74. if key.to_s.ends_with?("id")
  75. foreign_key_value
  76. else
  77. string_value
  78. end
  79. end
  80.  
  81. def assign_attributes_for_update!(hash, update_time)
  82. hash[:updated_on] = update_time if supports_updated_on?
  83. hash[:updated_at] = update_time if supports_updated_at?
  84. end
  85.  
  86. def assign_attributes_for_create!(hash, create_time = time_now)
  87. hash[:type] = model_class.name if supports_sti? && hash[:type].nil?
  88. hash[:created_on] = create_time if supports_created_on?
  89. hash[:created_at] = create_time if supports_created_at?
  90. assign_attributes_for_update!(hash, create_time)
  91. end
  92.  
  93. def supports_sti?
  94. @supports_sti_cache = model_class.column_names.include?("type") if @supports_sti_cache.nil?
  95. @supports_sti_cache
  96. end
  97.  
  98. def supports_created_on?
  99. if @supports_created_on_cache.nil?
  100. @supports_created_on_cache = (model_class.column_names.include?("created_on") && ActiveRecord::Base.record_timestamps)
  101. end
  102. @supports_created_on_cache
  103. end
  104.  
  105. def supports_updated_on?
  106. if @supports_updated_on_cache.nil?
  107. @supports_updated_on_cache = (model_class.column_names.include?("updated_on") && ActiveRecord::Base.record_timestamps)
  108. end
  109. @supports_updated_on_cache
  110. end
  111.  
  112. def supports_created_at?
  113. if @supports_created_at_cache.nil?
  114. @supports_created_at_cache = (model_class.column_names.include?("created_at") && ActiveRecord::Base.record_timestamps)
  115. end
  116. @supports_created_at_cache
  117. end
  118.  
  119. def supports_updated_at?
  120. if @supports_updated_at_cache.nil?
  121. @supports_updated_at_cache = (model_class.column_names.include?("updated_at") && ActiveRecord::Base.record_timestamps)
  122. end
  123. @supports_updated_at_cache
  124. end
  125.  
  126. def model_unique_keys(model)
  127. models[model]
  128. end
  129.  
  130. def models
  131. UNIQUE_INDEXES_FOR_MODELS
  132. end
  133.  
  134. def supports_remote_data_timestamp?(_)
  135. false
  136. end
  137.  
  138. def on_conflict_update
  139. false
  140. end
  141. end
  142.  
  143. def process_in_parallel(jobs, &block)
  144. require 'parallel'
  145. # Parallel.map(jobs, :in_threads => 8, &block)
  146. Parallel.map(jobs, :in_processes => 8, &block)
  147. end
  148.  
  149. ################################
  150. # Generate the mock data
  151. ################################
  152.  
  153. start_time = Time.now.utc
  154.  
  155. process_in_parallel(UNIQUE_INDEXES_FOR_MODELS.to_a) do |model_class, keys|
  156. ActiveRecord::Base.connection_pool.with_connection do
  157. (1..NUMBER_OF_DUPLICATES).each do |_|
  158. mock_saver = MockSaver.new(model_class, keys)
  159. data = []
  160. (1..NUMBER_OF_RECORDS).each do |i|
  161. data << mock_saver.build_data(model_class, i, "string_#{i}")
  162. if (i % BATCH_SIZE) == 0
  163. data_for_saving = data
  164. data = []
  165. else
  166. next
  167. end
  168.  
  169. mock_saver.get_connection.execute(
  170. mock_saver.build_insert_query(data_for_saving.first.keys, data_for_saving)
  171. )
  172. end
  173.  
  174. # Save remaining data
  175. unless data.blank?
  176. mock_saver.get_connection.execute(
  177. mock_saver.build_insert_query(data.first.keys, data)
  178. )
  179. end
  180. end
  181. end
  182. end
  183.  
  184. end_time = Time.now.utc
  185.  
  186. puts "Time taken #{end_time - start_time}s"
Add Comment
Please, Sign In to add comment