Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- #!/usr/bin/env ruby
- # Require REXML library to parse XML files
- require 'rexml/document'
- require 'rexml/streamlistener'
- include REXML
- # Parse command line arguments
- psql_path = "/usr/bin/psql"
- repo_path = "#{Dir.pwd}/etl"
- mode = ARGV[0]
- etl_source = ARGV[1]
- source_basename = File.basename(ARGV[1]).gsub(/.ktr/, '')
- etl_destination = "#{repo_path}/#{mode}/#{source_basename}_#{mode}.xml"
- environments = ['prod', 'stage', 'dev']
- end_script if mode.nil? or (mode != 'prod' and mode != 'stage' and mode != 'dev' and mode != 'ods' and mode != 'devdata' and mode != 'all') or etl_source.nil?
- if mode == 'all'
- environments.each do |envar|
- result = system("ruby deploy.rb #{envar} #{etl_source}")
- if result.nil?
- puts "Error running for all: #{$?}"
- elsif result
- puts "Successfully ran for #{envar}"
- end
- end
- exit
- end
- def self.end_script
- puts
- puts "Usage: ruby deploy.rb <prod|stage> <source_etl_file>"
- puts
- exit 1
- end
- # DB settings config
- db_config = [
- {:connection_name => 'dm_ods',
- :dev => {:server => '172.27.0.85', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'dm_ods'},
- :stage => {:server => '172.27.0.85', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'dm_ods'},
- :prod => {:server => '172.27.0.85', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'dm_ods'},
- :ods => {:server => '172.27.0.85', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'dm_ods'},
- :devdata => {:server => 'dev-data01.accend.io', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'mps', :port => '9432'}
- },
- {:connection_name => 'dm_etl',
- :dev => {:server => '172.27.0.85', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'dm_etl'},
- :stage => {:server => '172.27.0.85', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'dm_etl'},
- :prod => {:server => '172.27.0.85', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'dm_etl'},
- :ods => {:server => '172.27.0.85', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'dm_etl'},
- :devdata => {:server => 'dev-data01.accend.io', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'mps', :port => '9432'}
- },
- {:connection_name => 'mps',
- :dev => {:server => 'dev-psql01.drillmap.com', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'mps', :port => '9432'},
- :stage => {:server => 'stage-psql01.drillmap.com', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'mps', :port => '9432'},
- :prod => {:server => '172.27.0.46', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'mps', :port => '5432'},
- :devdata => {:server => 'dev-data01.accend.io', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'mps', :port => '9432'},
- :ods => {:server => 'localhost', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'mps', :port => '6432'}
- },
- {:connection_name => 'mysql_mps',
- :dev => {:server => '172.29.138.65', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'mps', :port => '3306'},
- :stage => {:server => '172.29.138.65', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'mps', :port => '3306'},
- :prod => {:server => '172.29.138.65', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'mps', :port => '3306'},
- :ods => {:server => '172.29.138.65', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'mps', :port => '3306'},
- :devdata => {:server => '172.29.138.65', :username => 'dataeng', :password => 'Rh@eRyNJmS', :database => 'mps', :port => '3306'}
- },
- {:connection_name => 'mssql_cop',
- :dev => {:server => 'cop-mssql01.drillmap.com', :username => 'DRMAPP', :password => 'Dj2u48jb!', :database => 'DrillMap_Dev', :port => '1433'},
- :stage => {:server => '<host>', :username => '<user>', :password => 'password', :database => 'datacopdb', :port => '1433'},
- :prod => {:server => '<host>', :username => '<user>', :password => '<password>', :database => '<database>', :port => '3306'},
- :ods => {:server => '<host>', :username => '<user>', :password => '<password>', :database => '<dtabase>', :port => '3306'},
- :ods => {:server => '<host>', :username => '<user>', :password => '<password>', :database => '<dtabase>', :port => '3306'}
- },
- {:connection_name => 'tbn_prod',
- :dev => {:server => 'sprddb03.petroleumplace.com', :username => 'wellsqry', :password => 'wellsqry', :database => 'WELLSP', :port => '1521'},
- :stage => {:server => 'sprddb03.petroleumplace.com', :username => 'wellsqry', :password => 'wellsqry', :database => 'WELLSP', :port => '1521'},
- :prod => {:server => 'sprddb03.petroleumplace.com', :username => 'wellsqry', :password => 'wellsqry', :database => 'WELLSP', :port => '1521'},
- }
- ];
- # Process file
- File.open(etl_source) do |source_stream|
- # open
- puts "Opening document ..."
- doc = Document.new(source_stream)
- # process
- puts "Processing document ..."
- doc.elements.each("transformation/step") do |step|
- step.elements.each("type") do |type|
- if type.text == "PGBulkLoader"
- step.elements.each("PsqlPath") {|path| path.text = psql_path}
- puts "PsqlPath changed to #{psql_path}"
- end
- end
- end
- doc.elements.each("transformation/connection") do |c|
- connection_name = ''
- c.elements.each("name") do |n|
- connection_name = n.text
- end
- if (dbc = db_config.detect { |h| h[:connection_name] == connection_name }).nil? or
- (the_config = dbc[mode.to_sym]).nil?
- p "Cannot find how to configure connection name #{connection_name}!"
- exit 1
- else
- puts "Configuring #{connection_name} ... "
- c.elements.each do |e|
- if the_config.has_key?(e.name.to_sym)
- e.text = the_config[e.name.to_sym]
- puts "Replaced value for #{e.name} with #{the_config[e.name.to_sym]}"
- end
- end
- end
- c.elements.each("attributes/attribute") do |attrib|
- attrib.elements.each("code") do |code|
- if code.text == "PORT_NUMBER"
- attrib.elements.each("attribute") {|port| port.text = the_config[:port]}
- end
- end
- end
- end
- # write
- puts "Writing document ..."
- formatter = REXML::Formatters::Default.new
- File.open(etl_destination, 'w') do |destination_stream|
- formatter.write(doc, destination_stream)
- end
- puts "Done ..."
- end
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement