Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- #!/usr/bin/python
- import os
- import subprocess
- import sys
- import datetime
- import shutil
- from datetime import datetime, timedelta
- import glob
- import string
- import re
- import shlex
- import smtplib
- import email
- #basic config - please do not change
- HOSTNAME = os.uname()[1]
- LOCKDIR = "/tmp/rn-innobackup.lock" #LOCKDIR
- LOGFILE = "/var/log/rn-mysqlbackup.log" #LOGFILE
- INNOBACKUPEX = "/usr/bin/innobackupex"
- ALERTMAIL = "mmaros@reflected.net" #to use multiple recepients, this needs to be turned into a list ["email1@bla.com", "email2@bla.com", "email3@bla.com", ...]
- #ALERTMAIL = "support@reflected.net" #to use multiple recepients, this needs to be turned into a list ["email1@bla.com", "email2@bla.com", "email3@bla.com", ...]
- #backup configuration
- #to use one binary over other, please put it at the start of the list, if it doesnt exist it will get skipped
- COMPRESS = ["/usr/bin/pbzip2","/usr/local/sbin/pbzip2","/usr/bin/pigz","/usr/local/sbin/pigz"]
- BACKUPDIR = '/home/backups/mysql/'
- RETENTION =int(7)
- STREAMING = False
- PARALEL = "4"
- USEMEMORY = "1G"
- CUSTOMOPT = "" #separate them with a space between, use syntax same as on the cli "--option=value --option2=value"
- COMPRESSOPT = ""
- ########Bunch of checks that need to be ran before we proceed
- ########
- #check which compression to use
- compression = ""
- for i in range(0, len(COMPRESS)):
- if os.path.isfile(COMPRESS[i]):
- compression = COMPRESS[i]
- break
- #exit if compression is not set up
- if compression == "":
- print ("none of compressions are available, please check which ones are available and which ones are provided")
- sys.exit()
- #compression being used:
- if ("pbzip2" in compression):
- backupSuffix = ".tar.bz2"
- elif ("pigz" in compression):
- backupSuffix = ".tar.gz"
- else:
- print ("wrong compression")
- sys.exit()
- #set backupname
- createDate = datetime.today().strftime("%Y%m%d-%H%M%S")
- if STREAMING:
- createName = "mysql_backups_" + HOSTNAME + "_" + createDate + "-streaming"
- createNameCompressed = "mysql_backups_" + HOSTNAME + "_" + createDate + "-streaming" + backupSuffix
- else:
- createName = "mysql_backups_" + HOSTNAME + "_" + createDate
- createNameCompressed = "mysql_backups_" + HOSTNAME + "_" + createDate + backupSuffix
- #check the mysql datadir
- commandToRun = "mysql -e 'select @@datadir;'"
- args = shlex.split(commandToRun)
- p = subprocess.Popen(args, stdout=subprocess.PIPE)
- DATADIR = p.stdout.read().splitlines()[-1]
- p.communicate()
- #print (output.splitlines()[-1])
- print (DATADIR)
- if os.path.isdir(DATADIR) is False:
- print (DATADIR + "does not exit")
- sys.exit()
- #check if it is slave
- commandToRun = "mysql -e 'show slave status\G'"
- args = shlex.split(commandToRun)
- p = subprocess.Popen(args, stdout=subprocess.PIPE)
- ISSLAVE = p.stdout.read()
- p.communicate()
- #if the machine is a slave add --slave-info to options
- if ISSLAVE:
- CUSTOMOPT += " --no-timestamp --slave-info "
- else:
- CUSTOMOPT += " --no-timestamp "
- print ("AAAAAAAAAAAAAAAAAAAAAAAAAAAAA " + CUSTOMOPT)
- #the fun starts here - Tuborg
- #skip disk check if .skipcheck is present in backupdir, remove after the run
- def skipCheck():
- if os.path.isfile(BACKUPDIR + ".skipcheck"):
- now = datetime.today().strftime("%Y-%m-%d %H:%M:%S")
- print(now + ": Skipping prerun disk space checks and running backups. skipcheck file will be removed")
- #function that runs backups
- os.remove(BACKUPDIR + ".skipcheck")
- runBackup()
- else:
- isRunnable()
- #check total freeDisk disk space on partition containting BACKUPDIR, in bytes
- def freeSpace(BACKUPDIR):
- st = os.statvfs(BACKUPDIR)
- freeDisk = int(st.f_bavail * st.f_frsize)
- return freeDisk
- #get the size of the backup specified
- def listBackups(BACKUPDIR, backupSuffix):
- backupNameRegex = os.path.join(BACKUPDIR + 'mysql_backups_' + HOSTNAME + '_' + '*' + backupSuffix)
- listBackups = glob.glob(backupNameRegex)
- return listBackups
- #checks the site of the latest backup
- def lastBackupSize():
- backupNames = listBackups(BACKUPDIR, backupSuffix)
- print (backupNames)
- if not backupNames:
- lastSize = 0
- else:
- lastSize = os.path.getsize(max(backupNames , key = os.path.getctime))
- return lastSize
- #get size of the mysql datadir
- def dirSize(path):
- total_size = 0
- for dirpath, dirnames, filenames in os.walk(path):
- for f in filenames:
- fp = os.path.join(dirpath, f)
- total_size += os.path.getsize(fp)
- return total_size
- #prerun checks
- def isRunnable():
- oldAge = 1
- lastSize = lastBackupSize()
- freeDisk = freeSpace(BACKUPDIR)
- #if streaming backup, we dont care about size of the datadir
- if STREAMING:
- datadirSize = 0
- else:
- datadirSize = dirSize(DATADIR)
- print("Size of last backup")
- print(lastSize)
- print("Free disk size")
- print(freeDisk)
- print("Datadir size")
- print(datadirSize)
- if ((freeDisk - lastSize - datadirSize) * 1.1) < (lastSize + datadirSize):
- msgBody = "Cant run backups as there is not enough free disk space"
- sendMail(msgBody)
- print("All gucci")
- runBackup()
- def removeOldBackups():
- #get list of all current backups
- listAllBackups = sorted(listBackups(BACKUPDIR, backupSuffix), reverse=True)
- print (listAllBackups)
- backupsToRemove = listAllBackups[RETENTION:]
- print (backupsToRemove)
- for i in backupsToRemove:
- print(i)
- os.remove(i)
- def runBackup():
- #how to split it
- #https://docs.python.org/2/library/subprocess.html
- #
- print("runBackup")
- if not STREAMING:
- commandToRun = "nice -n 19 " + INNOBACKUPEX + " " + CUSTOMOPT + "--parallel=" + PARALEL + " " + BACKUPDIR + createName
- args = shlex.split(commandToRun)
- with open("/tmp/innobackupex.stdout.log", "a+") as f_stdout, open("/tmp/innobackupex.stderr.log", "a+") as f_stderr:
- p = subprocess.Popen(args, stdout=f_stdout, stderr=f_stderr)
- p.communicate()
- #check if backup was successful
- lastline = file("/tmp/innobackupex.stderr.log", "r").readlines()[-1]
- if not re.search("completed OK!", lastline):
- msgBody = "Innobackupex failed, more information can be found in the log"
- sendMail(msgBody)
- print("innobackupex made it to the otherside")
- print("apply log now")
- #apply log to backup
- commandToRun = "nice -n 19 " + INNOBACKUPEX + " " + "--apply-log " + "--use-memory=" + USEMEMORY + " " + BACKUPDIR + createName
- print(commandToRun)
- args = shlex.split(commandToRun)
- print(args)
- with open("/tmp/apply.innobackupex.stdout.log", "a+") as f_stdout, open("/tmp/apply.innobackupex.stderr.log", "a+") as f_stderr:
- p = subprocess.Popen(args, stdout=f_stdout, stderr=f_stderr)
- p.communicate()
- #check if appylog was successful
- lastline = file("/tmp/apply.innobackupex.stderr.log", "r").readlines()[-1]
- if not re.search("completed OK!", lastline):
- msgBody = "Innobackupex failed at applying log, more information can be found in the log"
- sendMail(msgBody)
- #compress mysql configs as well
- commandToRun = "tar -cz /etc/mysql/"
- print (commandToRun)
- args = shlex.split(commandToRun)
- print(args)
- with open(BACKUPDIR + createName + "/etc-mysql.tar.gz", "a+") as f_stdout, open("/tmp/tar_error.log", "a+") as f_stderr:
- p = subprocess.Popen(args, stdout=f_stdout, stderr=f_stderr)
- p.communicate()
- #compress the backed up directory
- with open(BACKUPDIR + createNameCompressed, "a+") as f_stdout, open("/tmp/tar_error.log", "a+") as tar_stderr:
- tarItUp = "tar -v -c " + BACKUPDIR + createName
- argsTar = shlex.split(tarItUp)
- p1 = subprocess.Popen(argsTar, stdout=subprocess.PIPE, stderr=tar_stderr)
- compressItUp = "nice -n 19 " + compression + COMPRESSOPT + " -c "
- argsCompress = shlex.split(compressItUp)
- p2 = subprocess.Popen(argsCompress, stdin=p1.stdout, stdout=f_stdout, stderr=tar_stderr)
- p1.stdout.close() # Allow p1 to receive a SIGPIPE if p2 exits.
- p2.communicate()
- exitcodep2 = p2.returncode
- if exitcodep2 != 0:
- msgBody = "Compression of backup has failed. Please check"
- sendMail(msgBody)
- try:
- shutil.rmtree(BACKUPDIR + createName)
- except:
- msgBody = "Removal of the backupdir failed, please check. Tried to removed the following directory: " + BACKUPDIR + createName
- sendMail(msgBody)
- else:
- print("stuff")
- with open(BACKUPDIR + createNameCompressed, "a+") as f_stdout, open("/tmp/tar_error.log", "a+") as tar_stderr, open("/tmp/innobackupex.stderr.log", "a+") as inno_stderr:
- commandToRun = "nice -n 19 " + INNOBACKUPEX + " " + CUSTOMOPT + "--stream=tar" + " " + DATADIR
- argsCTR = shlex.split(commandToRun)
- print (argsCTR)
- print(commandToRun)
- p1 = subprocess.Popen(argsCTR, stdout=subprocess.PIPE, stderr=inno_stderr)
- compressItUp = compression + " " + COMPRESSOPT + " -c "
- argsCompress = shlex.split(compressItUp)
- p2 = subprocess.Popen(argsCompress, stdin=p1.stdout, stdout=f_stdout, stderr=tar_stderr)
- p1.stdout.close() # Allow p1 to receive a SIGPIPE if p2 exits.
- p2.communicate()
- lastline = file("/tmp/innobackupex.stderr.log", "r").readlines()[-1]
- if not re.search("completed OK!", lastline):
- msgBody = "Innobackupex failed on streaming backup, more information can be found in the log"
- sendMail(msgBody)
- removeOldBackups()
- #sendmail and exit function
- def sendMail(msgBody):
- sender = 'root@' + HOSTNAME + '.ded.reflected.net'
- receivers = ALERTMAIL
- msgBody = msgBody
- #print ("this works")
- #message = """From: """ + sender + """\nTo: """ + receivers + """\nSubject: backup failed on """ + HOSTNAME + """\n\n""" + msgBody + """\n"""
- #its still ugly, but ugly in mutliple lines
- message = ("From: " + sender + "\n"
- "To:" + receivers + "\n"
- "Subject: Backup failed on " + HOSTNAME + "\n\n"
- + msgBody + "\n")
- #smtpObj = smtplib.SMTP('smtp-out.reflected.net')
- #smtpObj.sendmail(sender, receivers, message)
- print ("Successfully sent email")
- sys.exit()
- if not os.path.exists(LOCKDIR):
- os.makedirs(LOCKDIR)
- skipCheck()
- else:
- print (LOCKDIR + " already exists, script exiting")
- sendMail(LOCKDIR + "already exists, backup can not start. Please check logs")
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement