Advertisement
Guest User

Untitled

a guest
May 27th, 2015
248
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. var fs = require('fs');
  2. var mysql = require('mysql');
  3. var path = require('path');
  4. var vow = require('vow');
  5. var spawn = require('child_process').spawn;
  6. var Metalib = require('fluent-ffmpeg').ffprobe;
  7. var os = require('os');
  8. var mv = require('mv');
  9. var nconf = require('nconf');
  10.  
  11.  
  12. nconf.file({file:'config.json'});
  13. nconf.defaults({
  14.   'host': 'localhost',
  15.   'user': 'root',
  16.   'password': 'root',
  17.   'database': 'test_db'
  18. });
  19.  
  20. function TaskManager(){
  21.   this.tasks = {};
  22. }
  23.  
  24. TaskManager.prototype.add = function(task){
  25.   if (this.tasks[task])
  26.     return;
  27.   this.tasks[task] = {
  28.     status: 'available',
  29.     task: task
  30.   };
  31. };
  32.  
  33. TaskManager.prototype.get = function(){
  34.   var keys = Object.keys(this.tasks);
  35.   if (!keys.length)
  36.     return null;
  37.   var tasks = this.tasks;
  38.   var availableTasks = keys.reduce(function(r,x){
  39.     if (tasks[x].status == 'available')
  40.       r.push(tasks[x]);
  41.     return r;
  42.   },[]);
  43.   if (!availableTasks.length)
  44.     return null;
  45.   var pick = availableTasks[0];
  46.   pick.status = 'blocked';
  47.   return pick;
  48. };
  49.  
  50. TaskManager.prototype.release = function(task){
  51.   if (this.tasks[task])
  52.   {
  53.     delete this.tasks[task];
  54.     return true;
  55.   }
  56.   return false;
  57. };
  58.  
  59.  
  60. var connection = mysql.createConnection({
  61.   host     : nconf.get('host'),
  62.   user     : nconf.get('user'),
  63.   password : nconf.get('password'),
  64.   database : nconf.get('database')
  65. });
  66.  
  67. function downloadVideo(video_id){
  68.   var dfd = vow.defer();
  69.   var tmpdir = os.tmpdir() + '/';
  70.   var dir = '../app/tube/video_quality/';
  71.   var filename = video_id + '.mp4';
  72.   var innerCounter = 0;
  73.   if (!fs.existsSync(dir+'lq/'+filename))
  74.   {
  75.     var lowQuality = spawn('youtube-dl', ['-f', '18/5', '-o', tmpdir + 'lq/%(id)s.%(ext)s',
  76.       'http://www.youtube.com/watch?v='+video_id]);
  77.  
  78.     lowQuality.on('exit', function(code, signal){
  79.       console.log('low quality exit', code, signal);
  80.       innerCounter++;
  81.       if (code === 1)
  82.       {
  83.         dfd.reject(code);
  84.       }
  85.       else
  86.       {
  87.         mv(tmpdir+'lq/'+filename, dir+'lq/'+filename, {mkdirp: true}, function(e){
  88.           if (e)
  89.           {
  90.             console.log(e);
  91.             process.exit(1);
  92.           }
  93.           dfd.resolve();
  94.         });
  95.       }
  96.     });
  97.   }
  98.   else
  99.   {
  100.     dfd.resolve();
  101.   }
  102.  
  103.   if (!fs.existsSync(dir+'hq/'+filename))
  104.   {
  105.     var highQuality = spawn('youtube-dl', ['-f', '22/135+140', '-o', tmpdir + 'hq/%(id)s.%(ext)s',
  106.       'http://www.youtube.com/watch?v='+video_id]);
  107.  
  108.     highQuality.on('exit', function(code, signal){
  109.       console.log('high quality exit');
  110.       innerCounter++;
  111.  
  112.       if (code === 1)
  113.       {
  114.         dfd.reject(code);
  115.       }
  116.       else
  117.       {
  118.         mv(tmpdir+'hq/'+filename, dir+'hq/'+filename, {mkdirp: true}, function(e){
  119.           if(e)
  120.           {
  121.             console.log(e);
  122.             process.exit(1);
  123.           }
  124.           dfd.resolve();
  125.         });
  126.       }
  127.     });
  128.   }
  129.   else
  130.   {
  131.     dfd.resolve();
  132.   }
  133.  
  134.   return dfd.promise();
  135.  
  136. }
  137.  
  138. function pushConvert(file){
  139.   convertTaskManager.add(file);
  140. }
  141.  
  142. function convertVideo(src){
  143.   console.log('start converting '+src);
  144.   var dfd = vow.defer();
  145.   var webmFormat = path.basename(src, '.mp4') + '.webm';
  146.   var dir = os.tmpdir() + '/';
  147.   if(fs.existsSync(src) && !fs.existsSync(dir+webmFormat))
  148.   {
  149.     var tmp = dir+webmFormat;
  150.   }
  151.   else
  152.   {
  153.     dfd.resolve();
  154.     return dfd.promise();
  155.   }
  156.   var filename = path.basename(tmp);
  157.   var srcDir = path.dirname(src) + '/';
  158.   var spawnConvert = spawn('ffmpeg', ['-i', src, '-c:v', 'libvpx', '-crf', '10', '-b:v', '1M', '-c:a', 'libvorbis', tmp]);
  159.   console.log('chain is moving forward', src, tmp);
  160.   listenProcess(spawnConvert);
  161.  
  162.   spawnConvert.on('exit', function(code, signal){
  163.     console.log('convert done', code, signal);
  164.     mv(tmp, srcDir + filename, {mkdirp: true}, function(){
  165.       dfd.resolve();
  166.     });
  167.   });
  168.   return dfd.promise();
  169. }
  170.  
  171.  
  172.  
  173. function listenProcess(ps){
  174.   ps.stderr.pipe(process.stdout);
  175.   ps.stdout.pipe(process.stdout);
  176. }
  177.  
  178. connection.connect();
  179.  
  180.  
  181. var selectVideos = function(connection){
  182.   connection.query('SELECT VIDEO_ID FROM TV_ITEM WHERE STATUS = 1 ORDER BY TV_ITEM_ID DESC LIMIT 5', function(err, rows, fields) {
  183.     var chain;
  184.    
  185.     rows.map(function(x){ return x.VIDEO_ID;}).forEach(function(video){
  186.       if (chain)
  187.       {
  188.         console.log('chain continues, video: ' + video);
  189.         chain = chain.then(downloadVideo.bind(null, video), function(err){
  190.             console.log('yt-dl err: ' + err);
  191.         });
  192.       }
  193.       else {
  194.         console.log('starting chain, video: ' + video);
  195.         chain = downloadVideo(video);
  196.       }
  197.     });
  198.  
  199.   });
  200. }
  201.  
  202. var convertTaskManager = new TaskManager();
  203. var downloadQueue = [];
  204.  
  205.  
  206. var convertWorker = {
  207.   state: 'idle',
  208.   work: function() {
  209.     console.log('convert worker state: ' + this.state);
  210.     console.log(convertTaskManager.tasks);
  211.     if (this.state != 'idle')
  212.       return;
  213.     this.state = 'check';
  214.     var item = convertTaskManager.get();
  215.     if (item == null)
  216.     {
  217.       this.state = 'idle';
  218.       return;
  219.     }
  220.     this.state = 'active';
  221.     this.item = item;
  222.     this.chain = this.convertItem();
  223.   },
  224.   convertItem: function(){
  225.     var dfd = vow.defer();
  226.     convertVideo(convertWorker.item.task).then(function(){
  227.       convertTaskManager.release(convertWorker.item.task);
  228.       convertWorker.item = convertTaskManager.get();
  229.       if (convertWorker.item)
  230.       {
  231.           convertWorker.chain.then(convertWorker.convertItem);
  232.       }
  233.       else
  234.       {
  235.         convertWorker.state = 'idle';
  236.         convertWorker.chain = null;
  237.       }
  238.       dfd.resolve();
  239.     });
  240.     return dfd.promise();
  241.   },
  242.   chain: null
  243. };
  244.  
  245. var downloadWorker = {
  246.  
  247.   state: 'idle',
  248.   work: function() {
  249.     if (this.state != 'idle')
  250.       return;
  251.     this.state = 'check';
  252.  
  253.     this.getVideos().then((function(){
  254.       console.log('download worker state: ' + this.state);
  255.       console.log(downloadQueue);
  256.       if (downloadQueue.length==0)
  257.       {
  258.         this.state = 'idle';
  259.         return;
  260.       }
  261.       this.state = 'active';
  262.       this.chain = this.checkNewVideo();
  263.     }).bind(downloadWorker));
  264.    
  265.    
  266.   },
  267.   getVideos: function(){
  268.     var dfd = vow.defer();
  269.     connection.query('SELECT VIDEO_ID FROM TV_ITEM WHERE STATUS = 1 AND DATA >= date_sub(NOW(), '
  270.     +'INTERVAL 2 MINUTE)', function(err, rows, fields) {
  271.       if (err) console.log(err);
  272.       console.log(rows.map(function(x){ return x.VIDEO_ID;}));
  273.       rows.map(function(x){ return downloadQueue.push(x.VIDEO_ID);});
  274.       dfd.resolve();
  275.     });
  276.     return dfd.promise();
  277.   },
  278.   checkNewVideo: function(){
  279.     var dfd = vow.defer();
  280.    
  281.     var id = downloadQueue.shift();
  282.     downloadVideo(id).then(function(){
  283.       if (downloadQueue.length)
  284.       {
  285.         downloadWorker.chain.then(downloadWorker.checkNewVideo);
  286.       }
  287.       else
  288.       {
  289.         downloadWorker.state = 'idle';
  290.         downloadWorker.chain = null;
  291.       }
  292.       dfd.resolve();
  293.     });
  294.     return dfd.promise();
  295.   },
  296.   chain: null
  297.  
  298. };
  299.  
  300.  
  301. setInterval(downloadWorker.work.bind(downloadWorker), 60*1000);
  302. setInterval(convertWorker.work.bind(convertWorker), 60*1000);
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement