Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- /**
- * Author: Toan Nguyen Dinh
- * Email: Toan@tabvn.com
- * Created: Sep 02, 2017
- */
- /**
- * Requires of modules
- */
- const fs = require('fs');
- const path = require('path');
- const spawn = require('child_process').spawn;
- const _ = require('lodash');
- const moment = require('moment');
- const tmp = require('tmp');
- tmp.setGracefulCleanup();
- /**
- * Amazone S3
- *
- */
- const AWS = require('aws-sdk');
- AWS.config.loadFromPath('aws.json');
- const s3 = new AWS.S3();
- /**
- * End Amazon S3
- */
- /**
- * End requires modules
- *
- */
- /**
- * Begin Express 4
- */
- const PORT = 3003;
- const express = require('express');
- const bodyParser = require('body-parser');
- const app = express();
- app.use(bodyParser.json()); // for parsing application/json
- app.use(bodyParser.urlencoded({extended: true})); // for parsing application/x-www-form-urlencoded
- /**
- * End of Express
- */
- /**
- *Begin Variables
- */
- const DATABASE_BACKUP = 'database';
- const CODE_BACKUP = 'code';
- const FULL = 'full';
- let production = false;
- let devConfig = {
- mongodb: 'dashboard',
- bucket: 'tom.tabvn.com'
- };
- let productionConfig = {
- mongodb: 'dashboard',
- bucket: 'producer.livex.tv'
- };
- let config = production ? productionConfig : devConfig;
- let backups = []; // array of processing backup
- let backupErrors = []; // Array items of backup error
- let restore = null;
- /**
- * End variables
- */
- let upload = (fileName, filePath, callback) => {
- let file = fs.createReadStream(filePath);
- let params = {Bucket: config.bucket, Key: fileName, Body: file};
- s3.putObject(params, function (err, data) {
- if (err) {
- console.log(err);
- if (callback) {
- return callback(err);
- }
- } else {
- if (callback) {
- return callback(null, data);
- }
- }
- });
- };
- let getBackupFiles = (callback) => {
- s3.listObjects({
- Bucket: config.bucket,
- MaxKeys: 100
- }, function (err, data) {
- if (callback) {
- return callback(err, data);
- }
- });
- };
- let deleteBackupFile = (key, callback) => {
- let params = {
- Bucket: config.bucket,
- Key: key
- };
- s3.deleteObject(params, function (err, data) {
- if (callback) {
- return callback(err, data);
- }
- });
- }
- let getDownloadUrl = (key = "", callback) => {
- let ONE_DAY = 60 * 60 * 24;
- let params = {
- Bucket: config.bucket,
- Key: key,
- Expires: ONE_DAY
- };
- s3.getSignedUrl('getObject', params, (err, data) => {
- if (err) {
- if (callback) {
- return callback(err);
- }
- }
- else {
- return callback(null, data);
- }
- });
- };
- let getBackupFileObject = (key = "", callback) => {
- let params = {
- Bucket: config.bucket,
- Key: key
- };
- s3.getObject(params, (err, data) => {
- if (callback) {
- return callback(err, data);
- }
- })
- };
- let compress = (pathToArchive, directoryPath, callback) => {
- let compressProcess = spawn('tar', ['zcvf', pathToArchive, '-C', directoryPath, '.']);
- compressProcess.on('exit', (code) => {
- if (code === 0) {
- if (callback) {
- return callback(null, true);
- }
- } else {
- let error = new Error("Error", code);
- if (callback) {
- return callback(error);
- }
- }
- })
- };
- let extract = (pathToArchive, directoryPath, callback) => {
- let extractProcess = spawn('tar', ['xvzf', pathToArchive, '-C', directoryPath]);
- extractProcess.on('exit', (code) => {
- if (code === 0) {
- if (callback) {
- return callback(null, true);
- }
- } else {
- let error = new Error("Error", code);
- if (callback) {
- return callback(error);
- }
- }
- });
- };
- let createFileName = (backup, ext = 'tar.gz') => {
- let names = [];
- let space = "---";
- let underSpace = '___';
- let snapshot = backup.snapshot ? backup.snapshot : "null";
- snapshot = _.replace(snapshot, /---/g, " ");
- snapshot = _.replace(snapshot, /___/g, " ");
- snapshot = _.replace(snapshot, /\//g, " ");
- snapshot = _.trim(snapshot);
- names.push('snapshot' + space + (snapshot));
- names.push('backupType' + space + backup.backupType);
- names.push('manually' + space + (backup.manually ? "true" : "false"));
- names.push('createdAt' + space + moment(backup.createdAt).unix());
- names.push('ext' + space + ext + underSpace);
- return _.join(names, underSpace) + '.' + ext;
- };
- let getObjectStructFromFileName = (filename) => {
- let space = "---";
- let underSpace = '___';
- let splitUnderScore = _.split(filename, underSpace);
- let snapshot = splitUnderScore && splitUnderScore[0] ? _.split(splitUnderScore[0], space) : null;
- let backupType = splitUnderScore && splitUnderScore[1] ? _.split(splitUnderScore[1], space) : null;
- let manually = splitUnderScore && splitUnderScore[2] ? _.split(splitUnderScore[2], space) : null;
- let createdAt = splitUnderScore && splitUnderScore[3] ? _.split(splitUnderScore[3], space) : null;
- let obj = {
- key: filename,
- snapshot: snapshot && snapshot[1] && snapshot[1] && snapshot[1] !== 'null' ? snapshot[1] : "",
- backupType: backupType && backupType[1] ? backupType[1] : null,
- manually: manually && manually[1] === 'true' ? true : false,
- createdAt: createdAt && createdAt[1] ? moment.unix(createdAt[1]).toDate() : null,
- size: 0,
- tag: null,
- };
- return obj;
- };
- let backupDatabase = (backup, callback) => {
- let tmpDirGenerate = tmp.dirSync({prefix: "livex-", unsafeCleanup: true});
- let tmpDir = tmpDirGenerate.name;
- let fileName = createFileName(backup, 'tar.gz');
- let dir = path.join(tmpDir, Date.now().toString());
- let arg = ['--db', config.mongodb, '--out', dir];
- let exportDatabaseProcess = spawn('mongodump', arg);
- let filePath = path.join(tmpDir, fileName);
- exportDatabaseProcess.on('exit', (code) => {
- if (code === 0) {
- compress(filePath, dir, (err, success) => {
- if (err) {
- if (callback) {
- return callback(err);
- }
- } else {
- upload(fileName, filePath, (err, data) => {
- // delete the file
- tmpDirGenerate.removeCallback();
- if (err) {
- if (callback) {
- return callback(err);
- }
- } else {
- if (callback) {
- return callback(null, data);
- }
- }
- });
- }
- });
- } else {
- console.log("Backup database error with code: ", code);
- if (callback) {
- return callback(new Error("An error backup with code:", code));
- }
- }
- })
- };
- let backupSourceCode = (backup) => {
- };
- let fullBackup = (backup) => {
- backupDatabase(backup, (err, success) => {
- console.log(err, success);
- });
- backupSourceCode(backup);
- };
- let doBackup = (backup) => {
- if (backup.backupType === DATABASE_BACKUP) {
- backupDatabase(backup, (err, success) => {
- console.log("backup database process:", err, success);
- // so let remove pending backup item.
- backups = _.filter(backups, (b) => b.id !== backup.id);
- if (err === null && success) {
- backups = _.filter(backups, (b) => b.id !== backup.id);
- } else {
- // if error so we do need add this backup item as error item. log it.
- backupErrors.push(backup);
- }
- });
- } else if (backup.backupType === CODE_BACKUP) {
- backupSourceCode(backup);
- } else {
- fullBackup(backup);
- }
- };
- let restoreDatabase = (backup, callback) => {
- let params = {
- Bucket: config.bucket,
- Key: backup.key
- };
- let tmpDirGenerate = tmp.dirSync({prefix: "livex-restore", unsafeCleanup: true});
- let tmpDownloadDir = tmpDirGenerate.name;
- let filePath = path.join(tmpDownloadDir, backup.key);
- let file = fs.createWriteStream(filePath);
- file.on('close', function () {
- // now need extract the database file
- extract(filePath, tmpDownloadDir, (err, success) => {
- if (err) {
- return callback(err);
- } else {
- // remove the file
- fs.unlinkSync(filePath);
- console.log("Begin restore the database");
- let arg = ['--db', config.mongodb, path.join(tmpDownloadDir, config.mongodb)];
- let restoreMongoProcess = spawn('mongorestore', arg);
- restoreMongoProcess.stderr.on('data', (data) => {
- });
- restoreMongoProcess.on('exit', (code) => {
- console.log("exit with code", code);
- if (code === 0) {
- tmpDirGenerate.removeCallback();
- return callback(null, true);
- } else {
- return callback(new Error("An error restore the database with code: ", code));
- }
- });
- }
- });
- });
- s3.getObject(params).createReadStream().on('error', function (err) {
- return callback(err);
- }).pipe(file);
- };
- let restoreSourceCode = (backup) => {
- };
- let fullRestore = (backup) => {
- };
- let doRestore = (key) => {
- let backupObject = getObjectStructFromFileName(key);
- restore = backupObject;
- if (backupObject.backupType === DATABASE_BACKUP) {
- restoreDatabase(backupObject, (err, success) => {
- console.log("The restore status: ", err, success);
- if (err === null && success) {
- // success
- restore = null;
- }
- });
- } else if (backupObject.backupType === CODE_BACKUP) {
- restoreSourceCode(backupObject);
- } else {
- fullRestore(backupObject);
- }
- };
- let handleError = (res, error = null, code = 503) => {
- return res.status(code).send(error);
- };
- /**
- * GET homepage.
- */
- app.get('/', function (req, res) {
- res.send('Livex Service Application.')
- });
- /**
- * GET router for /backups list all backups files from s3
- */
- app.get('/backups', (req, res, next) => {
- let backupItems = [];
- getBackupFiles((err, data) => {
- if (err) {
- return next(err);
- } else {
- if (typeof data !== 'undefined' && data !== null && typeof data.Contents !== 'undefined' && data.Contents !== null) {
- _.each(data.Contents, (item) => {
- // let destruct from item file name.
- let obj = getObjectStructFromFileName(item.Key);
- // we also need file size to display.
- obj.size = item.Size;
- // add ETag to object in case use later to request to s3
- obj.tag = _.replace(item.ETag, /"/g, "");
- backupItems.push(obj);
- });
- }
- // let sort the array, latest backups items will show first.
- backupItems.sort((a, b) => {
- let dateA = new Date(moment(a.createdAt));
- let dateB = new Date(moment(b.createdAt));
- return dateB - dateA;
- });
- return res.json(backupItems);
- }
- });
- });
- /**
- *
- * DELETE Router for /backups/:key
- */
- app.delete('/backups/:key', (req, res, next) => {
- if (typeof req.params.key !== "undefined" && req.params.key !== null) {
- let key = req.params.key;
- deleteBackupFile(key, (err, data) => {
- if (err) {
- return handleError(res, err, 404);
- } else {
- return res.json({success: true});
- }
- })
- } else {
- return handleError(res, 'Backup does not exist', 404);
- }
- });
- /**
- * GET download backup
- */
- app.get('/backups/:key/download', (req, res, next) => {
- if (typeof req.params.key !== "undefined" && req.params.key !== null) {
- let key = req.params.key;
- getDownloadUrl(key, (err, data) => {
- if (err) {
- return handleError(res, err);
- } else {
- return res.json({url: data});
- }
- });
- } else {
- return handleError(res, 'Backup does not exist', 404);
- }
- });
- /**
- * POST router for restore a backup
- */
- app.post('/backups/restore', (req, res) => {
- let data = req.body;
- let force = data.force ? data.force : false;
- if (restore !== null && !force) {
- // we dont allow restore white having restore processs.
- return handleError(res, 'Another restore process is running.', 503);
- }
- if (typeof data !== "undefined" && data !== null && data.key !== null) {
- let key = data.key;
- getBackupFileObject(key, (err, data) => {
- if (err) {
- return handleError(res, err, 404);
- } else {
- doRestore(key);
- return res.json({message: "Starting restore process."});
- }
- });
- } else {
- return handleError(res, 'Backup Does not exist', 404);
- }
- });
- /**
- * GET Router for /backups/pending
- * List all processing backups
- */
- app.get('/backups/pending', (req, res) => {
- return res.json(backups);
- });
- /**
- * GET router for /backups/errors
- * Log all backups as errors
- */
- app.get('/backups/errors', (req, res) => {
- return res.json(backupErrors);
- });
- /**
- * GET router /backups/errors/clear
- * Clear all errors logs
- */
- app.get('/backups/errors/clear', (req, res) => {
- backupErrors = [];
- return res.json(backupErrors);
- });
- /**
- * POST /backups/create
- * create new backup
- */
- app.post('/backups/create', function (req, res) {
- let data = req.body;
- if (typeof data === "undefined" || typeof data.id === "undefined" || data.id === null || typeof data.backupType === "undefined" || data.backupType === null || typeof data.manually === "undefined" || data.manually === null) {
- return handleError(res, "An error", 503);
- }
- let backup = {
- id: data.id ? data.id : null,
- snapshot: data.snapshot ? data.snapshot : "",
- manually: data.manually ? data.manually : false,
- backupType: data.backupType ? data.backupType : DATABASE_BACKUP,
- createdAt: data.createdAt ? data.createdAt : Date.now(),
- updatedAt: data.updatedAt ? data.updatedAt : Date.now(),
- status: "pending"
- };
- backups.push(backup);
- doBackup(backup);
- return res.send(req.body);
- });
- /**
- * Start app service
- */
- app.listen(PORT, function () {
- console.log('Service is running on port:', PORT)
- });
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement