Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- var fs = require('fs');
- var mysql = require('mysql');
- var path = require('path');
- var vow = require('vow');
- var spawn = require('child_process').spawn;
- var Metalib = require('fluent-ffmpeg').ffprobe;
- var os = require('os');
- var mv = require('mv');
- var nconf = require('nconf');
- nconf.file({file:'config.json'});
- nconf.defaults({
- 'host': 'localhost',
- 'user': 'root',
- 'password': 'root',
- 'database': 'test_db'
- });
- function TaskManager(){
- this.tasks = {};
- }
- TaskManager.prototype.add = function(task){
- if (this.tasks[task])
- return;
- this.tasks[task] = {
- status: 'available',
- task: task
- };
- };
- TaskManager.prototype.get = function(){
- var keys = Object.keys(this.tasks);
- if (!keys.length)
- return null;
- var tasks = this.tasks;
- var availableTasks = keys.reduce(function(r,x){
- if (tasks[x].status == 'available')
- r.push(tasks[x]);
- return r;
- },[]);
- if (!availableTasks.length)
- return null;
- var pick = availableTasks[0];
- pick.status = 'blocked';
- return pick;
- };
- TaskManager.prototype.release = function(task){
- if (this.tasks[task])
- {
- delete this.tasks[task];
- return true;
- }
- return false;
- };
- var connection = mysql.createConnection({
- host : nconf.get('host'),
- user : nconf.get('user'),
- password : nconf.get('password'),
- database : nconf.get('database')
- });
- function downloadVideo(video_id){
- var dfd = vow.defer();
- var tmpdir = os.tmpdir() + '/';
- var dir = '../app/tube/video_quality/';
- var filename = video_id + '.mp4';
- var innerCounter = 0;
- if (!fs.existsSync(dir+'lq/'+filename))
- {
- var lowQuality = spawn('youtube-dl', ['-f', '18/5', '-o', tmpdir + 'lq/%(id)s.%(ext)s',
- 'http://www.youtube.com/watch?v='+video_id]);
- lowQuality.on('exit', function(code, signal){
- console.log('low quality exit', code, signal);
- innerCounter++;
- if (code === 1)
- {
- dfd.reject(code);
- }
- else
- {
- mv(tmpdir+'lq/'+filename, dir+'lq/'+filename, {mkdirp: true}, function(e){
- if (e)
- {
- console.log(e);
- process.exit(1);
- }
- dfd.resolve();
- });
- }
- });
- }
- else
- {
- dfd.resolve();
- }
- if (!fs.existsSync(dir+'hq/'+filename))
- {
- var highQuality = spawn('youtube-dl', ['-f', '22/135+140', '-o', tmpdir + 'hq/%(id)s.%(ext)s',
- 'http://www.youtube.com/watch?v='+video_id]);
- highQuality.on('exit', function(code, signal){
- console.log('high quality exit');
- innerCounter++;
- if (code === 1)
- {
- dfd.reject(code);
- }
- else
- {
- mv(tmpdir+'hq/'+filename, dir+'hq/'+filename, {mkdirp: true}, function(e){
- if(e)
- {
- console.log(e);
- process.exit(1);
- }
- dfd.resolve();
- });
- }
- });
- }
- else
- {
- dfd.resolve();
- }
- return dfd.promise();
- }
- function pushConvert(file){
- convertTaskManager.add(file);
- }
- function convertVideo(src){
- console.log('start converting '+src);
- var dfd = vow.defer();
- var webmFormat = path.basename(src, '.mp4') + '.webm';
- var dir = os.tmpdir() + '/';
- if(fs.existsSync(src) && !fs.existsSync(dir+webmFormat))
- {
- var tmp = dir+webmFormat;
- }
- else
- {
- dfd.resolve();
- return dfd.promise();
- }
- var filename = path.basename(tmp);
- var srcDir = path.dirname(src) + '/';
- var spawnConvert = spawn('ffmpeg', ['-i', src, '-c:v', 'libvpx', '-crf', '10', '-b:v', '1M', '-c:a', 'libvorbis', tmp]);
- console.log('chain is moving forward', src, tmp);
- listenProcess(spawnConvert);
- spawnConvert.on('exit', function(code, signal){
- console.log('convert done', code, signal);
- mv(tmp, srcDir + filename, {mkdirp: true}, function(){
- dfd.resolve();
- });
- });
- return dfd.promise();
- }
- function listenProcess(ps){
- ps.stderr.pipe(process.stdout);
- ps.stdout.pipe(process.stdout);
- }
- connection.connect();
- var selectVideos = function(connection){
- connection.query('SELECT VIDEO_ID FROM TV_ITEM WHERE STATUS = 1 ORDER BY TV_ITEM_ID DESC LIMIT 5', function(err, rows, fields) {
- var chain;
- rows.map(function(x){ return x.VIDEO_ID;}).forEach(function(video){
- if (chain)
- {
- console.log('chain continues, video: ' + video);
- chain = chain.then(downloadVideo.bind(null, video), function(err){
- console.log('yt-dl err: ' + err);
- });
- }
- else {
- console.log('starting chain, video: ' + video);
- chain = downloadVideo(video);
- }
- });
- });
- }
- var convertTaskManager = new TaskManager();
- var downloadQueue = [];
- var convertWorker = {
- state: 'idle',
- work: function() {
- console.log('convert worker state: ' + this.state);
- console.log(convertTaskManager.tasks);
- if (this.state != 'idle')
- return;
- this.state = 'check';
- var item = convertTaskManager.get();
- if (item == null)
- {
- this.state = 'idle';
- return;
- }
- this.state = 'active';
- this.item = item;
- this.chain = this.convertItem();
- },
- convertItem: function(){
- var dfd = vow.defer();
- convertVideo(convertWorker.item.task).then(function(){
- convertTaskManager.release(convertWorker.item.task);
- convertWorker.item = convertTaskManager.get();
- if (convertWorker.item)
- {
- convertWorker.chain.then(convertWorker.convertItem);
- }
- else
- {
- convertWorker.state = 'idle';
- convertWorker.chain = null;
- }
- dfd.resolve();
- });
- return dfd.promise();
- },
- chain: null
- };
- var downloadWorker = {
- state: 'idle',
- work: function() {
- if (this.state != 'idle')
- return;
- this.state = 'check';
- this.getVideos().then((function(){
- console.log('download worker state: ' + this.state);
- console.log(downloadQueue);
- if (downloadQueue.length==0)
- {
- this.state = 'idle';
- return;
- }
- this.state = 'active';
- this.chain = this.checkNewVideo();
- }).bind(downloadWorker));
- },
- getVideos: function(){
- var dfd = vow.defer();
- connection.query('SELECT VIDEO_ID FROM TV_ITEM WHERE STATUS = 1 AND DATA >= date_sub(NOW(), '
- +'INTERVAL 2 MINUTE)', function(err, rows, fields) {
- if (err) console.log(err);
- console.log(rows.map(function(x){ return x.VIDEO_ID;}));
- rows.map(function(x){ return downloadQueue.push(x.VIDEO_ID);});
- dfd.resolve();
- });
- return dfd.promise();
- },
- checkNewVideo: function(){
- var dfd = vow.defer();
- var id = downloadQueue.shift();
- downloadVideo(id).then(function(){
- if (downloadQueue.length)
- {
- downloadWorker.chain.then(downloadWorker.checkNewVideo);
- }
- else
- {
- downloadWorker.state = 'idle';
- downloadWorker.chain = null;
- }
- dfd.resolve();
- });
- return dfd.promise();
- },
- chain: null
- };
- setInterval(downloadWorker.work.bind(downloadWorker), 60*1000);
- setInterval(convertWorker.work.bind(convertWorker), 60*1000);
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement