Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- #This script will be run via cron to check for messages from various applications
- #such as ntbackup, zmanda backup. these notifcation emmails will be parsed
- #and insterted into mysql for later usage for monitoring and rrd graphs
- require 'rmail'
- require 'active_record'
- require 'logger'
- require 'date'
- require 'time'
- #mbox containing the messages to be parsed
- mailbox="/var/spool/mail/monitor"
- #our log file
- log_file="/var/log/monitor_import"
- log = Logger.new(log_file)
- log.level = Logger::INFO
- log.datetime_format = "%Y-%m-%d %H:%M "
- #database settings and ActiveRecord models
- ActiveRecord::Base.establish_connection(:adapter=>"mysql",:host=>"localhost",:database=>"sciens_internal_dev",:username=>"user",:password=>"password")
- class Customer < ActiveRecord::Base
- has_many :customer_sites
- end
- class CustomerSite < ActiveRecord::Base
- has_many :customer_hosts
- end
- class CustomerHost < ActiveRecord::Base
- belongs_to :customer_site
- has_many :customer_ntb_datasets
- has_many :customer_zcb_datasets
- end
- class CustomerMessage < ActiveRecord::Base
- has_many :customer_ntb_jobs
- has_many :customer_zcb_jobs
- end
- class CustomerNtbDataset < ActiveRecord::Base
- belongs_to :customer_host
- has_many :customer_ntb_jobs
- end
- class CustomerNtbJob < ActiveRecord::Base
- belongs_to :customer_ntb_dataset
- belongs_to :customer_message
- end
- class CustomerZcbDataset < ActiveRecord::Base
- belongs_to :customer_host
- has_many :customer_zcb_jobs
- end
- class CustomerZcbJob < ActiveRecord::Base
- belongs_to :customer_zcb_dataset
- belongs_to :customer_message
- end
- #load all messages from mbox and insert them into an array
- log.info "loading messages from #{mailbox}"
- messages = Array.new
- File.open(mailbox) do |mbox|
- RMail::Mailbox.parse_mbox(mbox) do |raw|
- messages.push(raw)
- end
- end
- #Main Loop - check to see if message has already been parsed and inserted into DB
- messages.each do |message|
- if( (message_id = /^Message-ID:\s\<(.*)\>/i.match(message) ) && (customer_message = CustomerMessage.find_by_message_id(message_id[1])) )
- elsif(message_id = /^Message-ID:\s\<(.*)\>/i.match(message))
- customer_message = CustomerMessage.create(:message_id=>message_id[1],:body=>message)
- #this looks like a new message, lets get the host, customer
- if( (site = /^From\s*(?:[0-9a-z\-_]*\.)?([0-9a-z\-_]*)\@/i.match(message)) && (host = /^Subject:\s*(?:nt-backup log host:\s*|\[zcb:\s*)([0-9a-z_\-]*)/i.match(message)) )
- #does this customer site exist?
- if (!customer_site = CustomerSite.find_by_name(site[1]))
- if(!customer=Customer.find_by_name(site[1]))
- #customer does not exist, lets create this first
- customer=Customer.create(:name=>site[1],:description=>"Created by monitor_import")
- log.info "Created customer: #{site[1]}"
- customer_site = customer.customer_sites.create(:name=>site[1],:description=>"Created by monitor_import")
- log.info "Created customer-site: #{site[1]}"
- else
- #customer does exist, lets place the site under this customer
- customer_site = customer.customer_sites.create(:name=>site[1],:description=>"Created by monitor_import")
- log.info "Created customer: #{site[1]}"
- end
- end
- #does host exist? if not lets create it
- if(!customer_host = customer_site.customer_hosts.find_by_name(host[1]))
- customer_host = customer_site.customer_hosts.create(:name=>host[1], :description=>"Created by monitor_import")
- log.info "Created customer host #{host[1]}"
- else
- log.info "Customer host exists #{host[1]}"
- end
- end
- #We should now have customer, customer site and customer host
- #We now need to determine what to do with this message by looking at the subject line
- #zmanda cloud backup
- if(/^Subject:\s*\[zcb:/i.match(message))
- #Pick dataset name from message
- if (job_info = /(^Backup|^Upload) of backup set "(.*)" (?:is )?(failed|successful)/.match(message))
- dataset=job_info[2]
- task=job_info[1]
- status=job_info[3]
- #Check to see if this dataset is already in the customer_zcb_datasets table
- if(customer_dataset = customer_host.customer_zcb_datasets.find_by_name(dataset))
- else
- #create this dataset
- customer_dataset = customer_host.customer_zcb_datasets.create(:name=>dataset,:description=>"Created by zmanda_import")
- log.info "Created dataset: #{customer_dataset[:name]} withd id: #{customer_dataset[:id]}, for host #{customer_host[:name]}"
- end
- else
- log.warn "Something went wrong, unable to find dataset in message from #{from_address}"
- end
- #determine if this is an Upload or a Backup job
- if (task == "Backup")
- #gather backup job details
- start_time = /^Backup start time : (\d{4}\/\d\d\/\d\d) (\d\d:\d\d:\d\d)/.match(message)
- end_time = /^Backup end time : (\d{4}\/\d\d\/\d\d) (\d\d:\d\d:\d\d)/.match(message)
- files_on_disk = /^Files on disk : (\d*)/.match(message)
- files_in_backup = /^Files in backup : (\d*)/.match(message)
- details = /^Error Details : (.*)/.match(message)
- backup_size = /^Total Bytes backed up : (\d.*) (KB|MB|GB)/.match(message)
- #convert all sizes to KB
- size = 0
- if (backup_size && backup_size.length > 1)
- size = backup_size[1]
- size = size.to_f
- size = size * 1024 if backup_size[2] == "MB"
- size = size * 1024 * 1024 if backup_size[2] == "GB"
- size = size.to_i
- end
- #format time stamps for db format is YYYY/MM/DD HH:MM:SS
- start_datetime = start_time[1] << " " << start_time[2]
- end_datetime = end_time[1] << " " << end_time[2]
- #add code here to calcuate backup rate, may not be possible if we dont have compression sizes?
- zcb_job = customer_dataset.customer_zcb_jobs.create(
- :task=>"Backup",
- :size=>size,
- :start_time=>start_datetime,
- :end_time=>end_datetime,
- :files_on_disk=>files_on_disk[1],
- :files_in_backup=>files_in_backup[1],
- :details=>details[1],
- :customer_message_id => customer_message[:id],
- :status=>status
- )
- log.info "Imported Zmanda backup job with for #{customer_dataset[:name]}, Status: #{zcb_job[:status]}, for host: #{customer_host[:name]}"
- elsif (task == "Upload")
- #gather upload job details
- start_time = /^Upload start time : (\d{4}\/\d\d\/\d\d) (\d\d:\d\d:\d\d)/.match(message)
- end_time = /^Upload end time : (\d{4}\/\d\d\/\d\d) (\d\d:\d\d:\d\d)/.match(message)
- details = /^Error Details : (.*)/.match(message)
- upload_rate = /^Upload Rate : (\d*) Kbps/.match(message)
- upload_size = /^Bytes uploaded : ([\d\.]*) (KB|MB|GB)/.match(message)
- #convert all sizes to KB
- if (upload_size && upload_size.length > 1)
- size = upload_size[1]
- size = size.to_f
- size = size * 1024 if upload_size[2] == "MB"
- size = size * 1024 * 1024 if upload_size[2] == "GB"
- size = size.to_i
- end
- #format time stamps for db format is YYYY/MM/DD HH:MM:SS
- start_datetime = start_time[1] << " " << start_time[2]
- end_datetime = end_time[1] << " " << end_time[2]
- zcb_job = customer_dataset.customer_zcb_jobs.create(
- :task=>"Upload",
- :size=>size,
- :start_time=>start_datetime,
- :end_time=>end_datetime,
- :rate=>upload_rate[1],
- :size=>size,
- :details=>details[1],
- :customer_message_id => customer_message[:id],
- :status=>status
- )
- log.info "Imported Zmanda upload job with for #{customer_dataset[:name]}, Status: #{zcb_job[:status]}, for host: #{customer_host[:name]}"
- #match upload with backup job, for now will assume it was the previous backup for this dataset
- else
- #not sure what job type this is break out here
- log.warn "Unkown job type - #{task}"
- end
- #nt backup
- elsif(/^Subject:\s*nt-backup/i.match(message))
- #Pick dataset names from message
- datasets=message.scan(/^Backup .* of \"(.*)\"/)
- backups=message.scan(/^Backup .* of \"(.*)\"\n^Backup set (.*)\n^Backup description: \"(.*)\"\n^Media name: \"(.*)\"\n\n^Backup Type: (.*)\n\n^Backup started on (.*) at (.*) (AM|PM).\n^Backup completed on (.*) at (.*) (AM|PM).\n^Directories: (.*)\n^Files: (.*)\n^Bytes: (.*)/)
- #did this job finish without error?
- if (status = /^NTBackup finished the backup with no errors./.match(message))
- job_status="successful"
- else
- job_status="failed"
- end
- #backups array layout, format location) description - example
- # 0) Dataset name - E:
- # 1) Backup set - #3 on media #1
- # 2) Backup Descrition - SBS Backup created on 1/26/2011 at 11:00 PM
- # 3) Media Name - Media created 1/26/2011 at 11:00 PM
- # 4) Backup Type - Normal
- # 5) Backup Start date - 1/27/2011
- # 6) Backup Start time - 1:41
- # 7) Backup Start time am or pm? - AM
- # 8) Backup End date - 1/27/2011
- # 9) Backup End time - 1:43
- #10) Backup End time am or pm? - AM
- #11) Directories - 71
- #12) Files - 895
- #13) Bytes - 281,723,426
- #check to see if datasets already exist, if not create them
- if(datasets)
- 0.upto((datasets.count)-1) {|i|
- #does dataset already exist?
- if(customer_dataset = customer_host.customer_ntb_datasets.find_by_name(datasets[i][0]))
- elsif(customer_dataset = customer_host.customer_ntb_datasets.create(:name=>datasets[i][0]))
- log.info "Created NT-Backup dataset #{customer_dataset[:name]} with id of : #{customer_dataset[:id]} "
- else
- end
- }
- end
- #insert each ntbackup job
- backups.each do |job|
- #this size == 14 shit needs to go
- if(job.size ==14 && customer_dataset=customer_host.customer_ntb_datasets.find_by_name(job[0]))
- backupset=job[1]
- description=job[2]
- media_name=job[3]
- type=job[4]
- start_time=DateTime.parse(job[5] job[6] job[7])
- end_time=DateTime.parse(job[8] job[9] job[10])
- directories=job[11].gsub(",","").to_i
- files=job[12].gsub(",","").to_i
- size=job[13].gsub(",","").to_i/1024
- customer_job = customer_dataset.customer_ntb_jobs.create(
- :start_time=>start_time,
- :end_time=>end_time,
- :size=>size,
- :directories=>directories,
- :files=>files,
- :status=>job_status,
- :backupset=>backupset,
- :description=>description,
- :media_name=>media_name,
- :customer_message_id=>customer_message[:id],
- :type=>type
- )
- end
- end
- end
- end
- end #end of Main Loop
- log.info "Application Clean Exit"
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement