Advertisement
Guest User

Untitled

a guest
Oct 6th, 2016
90
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 6.45 KB | None | 0 0
  1. #!/usr/bin/env ruby
  2.  
  3. # Require REXML library to parse XML files
  4.  
  5. require 'rexml/document'
  6. require 'rexml/streamlistener'
  7. include REXML
  8.  
  9.  
  10. # Parse command line arguments
  11. psql_path = "/usr/bin/psql"
  12. repo_path = "#{Dir.pwd}/etl"
  13. mode = ARGV[0]
  14. etl_source = ARGV[1]
  15. source_basename = File.basename(ARGV[1]).gsub(/.ktr/, '')
  16. etl_destination = "#{repo_path}/#{mode}/#{source_basename}_#{mode}.xml"
  17. environments = ['prod', 'stage', 'dev']
  18.  
  19. end_script if mode.nil? or (mode != 'prod' and mode != 'stage' and mode != 'dev' and mode != 'ods' and mode != 'devdata' and mode != 'all') or etl_source.nil?
  20.  
  21. if mode == 'all'
  22. environments.each do |envar|
  23. result = system("ruby deploy.rb #{envar} #{etl_source}")
  24. if result.nil?
  25. puts "Error running for all: #{$?}"
  26. elsif result
  27. puts "Successfully ran for #{envar}"
  28. end
  29. end
  30. exit
  31. end
  32.  
  33. def self.end_script
  34. puts
  35. puts "Usage: ruby deploy.rb <prod|stage> <source_etl_file>"
  36. puts
  37. exit 1
  38. end
  39.  
  40. # DB settings config
  41. db_config = [
  42. {:connection_name => 'dm_ods',
  43. :dev => {:server => '172.27.0.85', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'dm_ods'},
  44. :stage => {:server => '172.27.0.85', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'dm_ods'},
  45. :prod => {:server => '172.27.0.85', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'dm_ods'},
  46. :ods => {:server => '172.27.0.85', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'dm_ods'},
  47. :devdata => {:server => 'dev-data01.accend.io', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'mps', :port => '9432'}
  48. },
  49. {:connection_name => 'dm_etl',
  50. :dev => {:server => '172.27.0.85', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'dm_etl'},
  51. :stage => {:server => '172.27.0.85', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'dm_etl'},
  52. :prod => {:server => '172.27.0.85', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'dm_etl'},
  53. :ods => {:server => '172.27.0.85', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'dm_etl'},
  54. :devdata => {:server => 'dev-data01.accend.io', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'mps', :port => '9432'}
  55. },
  56. {:connection_name => 'mps',
  57. :dev => {:server => 'dev-psql01.drillmap.com', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'mps', :port => '9432'},
  58. :stage => {:server => 'stage-psql01.drillmap.com', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'mps', :port => '9432'},
  59. :prod => {:server => '172.27.0.46', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'mps', :port => '5432'},
  60. :devdata => {:server => 'dev-data01.accend.io', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'mps', :port => '9432'},
  61. :ods => {:server => 'localhost', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'mps', :port => '6432'}
  62. },
  63. {:connection_name => 'mysql_mps',
  64. :dev => {:server => '172.29.138.65', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'mps', :port => '3306'},
  65. :stage => {:server => '172.29.138.65', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'mps', :port => '3306'},
  66. :prod => {:server => '172.29.138.65', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'mps', :port => '3306'},
  67. :ods => {:server => '172.29.138.65', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'mps', :port => '3306'},
  68. :devdata => {:server => '172.29.138.65', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'mps', :port => '3306'}
  69. },
  70. {:connection_name => 'mssql_cop',
  71. :dev => {:server => 'cop-mssql01.drillmap.com', :username => 'DRMAPP', :password => 'Dj2u48jb!', :database => 'DrillMap_Dev', :port => '1433'},
  72. :stage => {:server => '<host>', :username => '<user>', :password => 'password', :database => 'datacopdb', :port => '1433'},
  73. :prod => {:server => '<host>', :username => '<user>', :password => '<password>', :database => '<database>', :port => '3306'},
  74. :ods => {:server => '<host>', :username => '<user>', :password => '<password>', :database => '<dtabase>', :port => '3306'},
  75. :ods => {:server => '<host>', :username => '<user>', :password => '<password>', :database => '<dtabase>', :port => '3306'}
  76. },
  77. {:connection_name => 'tbn_prod',
  78. :dev => {:server => 'sprddb03.petroleumplace.com', :username => 'wellsqry', :password => 'wellsqry', :database => 'WELLSP', :port => '1521'},
  79. :stage => {:server => 'sprddb03.petroleumplace.com', :username => 'wellsqry', :password => 'wellsqry', :database => 'WELLSP', :port => '1521'},
  80. :prod => {:server => 'sprddb03.petroleumplace.com', :username => 'wellsqry', :password => 'wellsqry', :database => 'WELLSP', :port => '1521'},
  81. }
  82. ];
  83.  
  84.  
  85. # Process file
  86.  
  87. File.open(etl_source) do |source_stream|
  88.  
  89. # open
  90. puts "Opening document ..."
  91. doc = Document.new(source_stream)
  92.  
  93. # process
  94. puts "Processing document ..."
  95. doc.elements.each("transformation/step") do |step|
  96. step.elements.each("type") do |type|
  97. if type.text == "PGBulkLoader"
  98. step.elements.each("PsqlPath") {|path| path.text = psql_path}
  99. puts "PsqlPath changed to #{psql_path}"
  100. end
  101. end
  102. end
  103. doc.elements.each("transformation/connection") do |c|
  104. connection_name = ''
  105. c.elements.each("name") do |n|
  106. connection_name = n.text
  107. end
  108. if (dbc = db_config.detect { |h| h[:connection_name] == connection_name }).nil? or
  109. (the_config = dbc[mode.to_sym]).nil?
  110. p "Cannot find how to configure connection name #{connection_name}!"
  111. exit 1
  112. else
  113. puts "Configuring #{connection_name} ... "
  114. c.elements.each do |e|
  115. if the_config.has_key?(e.name.to_sym)
  116. e.text = the_config[e.name.to_sym]
  117. puts "Replaced value for #{e.name} with #{the_config[e.name.to_sym]}"
  118. end
  119. end
  120. end
  121. c.elements.each("attributes/attribute") do |attrib|
  122. attrib.elements.each("code") do |code|
  123. if code.text == "PORT_NUMBER"
  124. attrib.elements.each("attribute") {|port| port.text = the_config[:port]}
  125. end
  126. end
  127. end
  128. end
  129.  
  130. # write
  131. puts "Writing document ..."
  132. formatter = REXML::Formatters::Default.new
  133. File.open(etl_destination, 'w') do |destination_stream|
  134. formatter.write(doc, destination_stream)
  135. end
  136.  
  137. puts "Done ..."
  138.  
  139. end
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement