daily pastebin goal
58%
SHARE
TWEET

Untitled

a guest Jul 24th, 2016 728 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. #!/usr/bin/node
  2.  
  3. //
  4.  
  5. //mock file index design
  6. ({
  7.   stat:{},
  8.   '/':{}
  9. })
  10.  
  11.  
  12.  
  13.  
  14.  
  15.  
  16.  
  17. //
  18.  
  19. fs=require('fs');
  20. child_process=require('child_process');
  21.  
  22. var argsRules = {
  23.   '--':true,
  24.   boolean:[
  25.     'help',
  26.     'verbose',
  27.     'deep',
  28.     'fast',
  29.     'careless',
  30.     'interactive',
  31.     'halfinteractive',
  32.     'all',
  33.     'simulate',
  34.     'export',
  35.     'init',
  36.     'md5sumcorrupted',
  37.     'checkindex',
  38.     'forceupdate',
  39.     'legacyconvert',
  40.     'backup'
  41.   ],
  42.   string:[
  43.     'mountpoint',
  44.     'indexdir',
  45.     'importcorrupted'
  46.   ],
  47.   alias:{
  48.     help:['h'],
  49.     verbose:['v'],
  50.     deep:['d'],
  51.     fast:['f'],
  52.     careless:['L'],
  53.     interactive:['i'],
  54.     halfinteractive:['I'],
  55.     all:['a'],
  56.     simulate:['s'],
  57.     export:['e'],
  58.     importcorrupted:['import'],
  59.     init:['g'],
  60.     backup:['b'],
  61.     mountpoint:['mnt','M']
  62.   },
  63.   description:{
  64.     help:'show this help',
  65.     verbose:'be verbose',
  66.     deep:'force md5 recheck',
  67.     fast:'force md5 skip',
  68.     careless:'don\'t recheck md5 if only less important stat values changed (uid,gid,permissions)',
  69.     interactive:'enable interactive mode',
  70.     halfinteractive:'enable interactive mode only if index changed',
  71.     mountpoint:'specify default drives mountpoint',
  72.     indexdir:'specify default index dir prefix',
  73.     all:'scan default drives mount dir searching for all available indexes',
  74.     simulate:'enable simulation mode (do modify/write any files)',
  75.     export:'export md5 hashes in md5sum -c readable format for faster validation than --deep',
  76.     importcorrupted:'import list of files for force md5 validation',
  77.     md5sumcorrupted:'read importcorrupted file as md5sum -c output',
  78.     init:'create new index from scratch',
  79.     checkindex:'Validate index sha512 before parsing',
  80.     forceupdate:'update extindex file even if there were no changes noted',
  81.     legacyconvert:'read file in legacy mode (do not expect first 128 bytes to be sha512)',
  82.     backup:'number of backup copies to keep'
  83.   },
  84.   default:{
  85.     verbose:true,
  86.     halfinteractive:true,
  87.     mountpoint:'/dmnt',
  88.     indexdir:'/etc/md5index',
  89.     checkindex:true,
  90.     backup:3
  91.   }
  92. }
  93.  
  94. args=require('/usr/lib/node_modules/minimist/')(process.argv.slice(2),argsRules);
  95.  
  96. (function(){
  97.   var validArgs=[].concat.apply(['--','_'],argsRules.boolean.concat(argsRules.string).map(function(key){return [key].concat((key in argsRules.alias)?argsRules.alias[key]:[])})),
  98.       invalidArgs=[];
  99.   Object.keys(args).forEach(function(key){validArgs.indexOf(key)==-1&&invalidArgs.push(key)})
  100.   if (invalidArgs.length){
  101.     console.error('invalidArgs: \n'+invalidArgs.join('\n'));
  102.     process.exit(7);
  103.   }
  104. })()
  105.  
  106. if (args.help){
  107.   var keys = Object.keys(argsRules.description),white = '',
  108.       args = Object.keys(argsRules.description).map(function(arg,i,arr){return '  '+((arg.length>1?'--':'-')+arg+',  '+(argsRules.alias[arg]?(argsRules.alias[arg].map(function(a){return (a.length>1?'--':'-')+a}).join(', ')):''))+'        '});
  109.   args.forEach(function(a){if (a.length>white.length) white=a.split('').map(function(){return ' '}).join('')})
  110.   console.log('\n'+args.map(function(a,i){return a+white.substr(a.length)+argsRules.description[keys[i]]+((keys[i] in argsRules.default)?('  [DEFAULT:'+argsRules.default[keys[i]]+']'):'')}).join('\n'));
  111.   process.exit(0);
  112. }
  113.  
  114. if (args.init)
  115.   args.forceupdate=true;
  116.  
  117. if (args.legacyconvert){
  118.   args.checkindex=false,
  119.   args.forceupdate=true;
  120. }
  121.  
  122. errors=[];
  123.  
  124. global.missing={
  125.   file:[],
  126.   index:[]
  127. }
  128.  
  129. global.diff=[]
  130.  
  131. indexes=(args.all?
  132.   (!fs.existsSync(args.mountpoint)?(function(){throw 'Could not find default drives location'})():
  133.   [].concat.apply([],fs.readdirSync(args.mountpoint).map(function(p){
  134.     return args.mountpoint+'/'+p+'/'+args.indexdir;
  135.   }).filter(function(p){
  136.     return fs.existsSync(p)&&fs.statSync(p).isDirectory();
  137.   }).map(function(p){
  138.     return fs.readdirSync(p).filter(function(f){
  139.       return f.match(/.*\.extindex$/)
  140.     }).map(function(f){
  141.       console.log('Found: ',p+'/'+f);
  142.       return p+'/'+f
  143.     })
  144.   }))):
  145. args._).filter(function(p,i){
  146.   if (args.init^(!fs.existsSync(p))){
  147.     console.error('Index '+(args.init?'already exists':'does not exist')+' - aborting',p);
  148.     process.exit(10);
  149.   }
  150.   if (args.init&&!fs.existsSync(args['--'][i])){
  151.     console.error('Could not find base dir / source md5index for '+p)
  152.     process.exit(13);
  153.   }
  154.   return true;
  155. }).map(function(p,i){
  156.   if (!args.init){
  157.     args.verbose&&console.log('Reading index... (',p,')');
  158.     var bfile = fs.readFileSync(fs.realpathSync(p)),
  159.         sha512 = bfile.slice(0,128),
  160.         content = bfile.slice(128);
  161.        
  162.     var sha = child_process.spawnSync('sha512sum',[],{input:content})
  163.     if (sha.error||sha.status){
  164.       console.error(sha.error||'sha512sum exit status code: '+sha.status+' '+String(sha.stderr));
  165.       process.exit(45);
  166.     }
  167.    
  168.     if (String(sha.stdout).substr(0,128)!=sha512){
  169.       if (args.checkindex){
  170.         console.error('Index sha512 check failed - index is probably damaged - aborting\n',sha512,'\n',String(sha.stdout).substr(0,128))
  171.         process.exit(14);
  172.       } else
  173.         console.warn('Index sha512sum check failed - index is probably damaged - forced ignore\n',sha512,'\n',String(sha.stdout).substr(0,128));
  174.     }
  175.     try{
  176.       var pind = JSON.parse(args.legacyconvert?bfile:content),
  177.           ostr = String(args.legacyconvert?bfile:content),
  178.           str = JSON.stringify(pind,null,2);
  179.          
  180.       for (var i = 0, l = str.length ; i < l ; i++)
  181.         if (str[i]!=ostr[i]){
  182.           console.error('Soft bug occured - it\'s serious bug and probably classifies as node bug or linux memcache bug. Should be reported');
  183.           throw ('Original string and reparsed don\'t match at '+i+' byte - system string conversion malfunction - abtorting')
  184.         }
  185.      
  186.       if (!(typeof pind == 'object' && !Array.isArray(pind) && 'index' in pind))
  187.         throw 'Invalid index file';
  188.      
  189.       var curpath = fs.realpathSync(p);
  190.       if (pind.extindex!=curpath)
  191.         console.warn('Index has been moved from its previous location - it may result in errors');
  192.       pind.extindex=curpath;
  193.      
  194.       if (!pind.index){
  195.         pind.index=[];
  196.         for (var i = 0, l = pind.subindexnum ; i < l ; ++i)
  197.           pind.index.push(pind.extindex+'.part'+i);
  198.        
  199.         pind.index=[].concat.apply([],pind.index.map(function(p,i){
  200.           args.verbose&&console.log('Reading subindex... (',p,')');
  201.           if (!fs.existsSync(p))
  202.             throw 'Expected subindex does not exist';
  203.          
  204.           var bfile = fs.readFileSync(p),
  205.               sha512 = bfile.slice(0,128),
  206.               content = bfile.slice(128);
  207.          
  208.           var sha = child_process.spawnSync('sha512sum',[],{input:content});
  209.           if(sha.error||sha.status)
  210.             throw sha.error||'sha512sum exit status code for subindex '+i+': '+sha.status+' '+String(sha.stderr);
  211.           if((sha=String(sha.stdout).substr(0,128))!=sha512){
  212.             if (args.checkindex)
  213.               throw 'Subindex sha512 check failed - subindex is probably damaged - aborting\n'+sha512+'\n'+sha;
  214.             else
  215.               console.warn('Subindex sha512 check failed - subindex is probably damaged - forced ignore\n'+sha512+'\n'+sha);
  216.           }
  217.          
  218.           var sind = JSON.parse(content),
  219.               ostr=String(content),
  220.               str = JSON.stringify(sind,null,2);
  221.              
  222.           for (var i = 0, l = str.length; i < l ; i++)
  223.             if (str[i]!=ostr[i]){
  224.               console.error('Soft bug occured - it\'s serious bug and probably classifies as node bug or linux memcache bug. Should be reported');
  225.               throw ('Original string and reparsed don\'t match at '+i+' byte - system string conversion malfunction - abtorting')
  226.             }
  227.            
  228.           return sind;
  229.          
  230.         }));
  231.       }
  232.      
  233.       return pind;
  234.     } catch (e) {
  235.       console.error('Could not read index - aborting',p,e);
  236.       process.exit(11);
  237.     }
  238.   } else {
  239.     args.verbose&&console.log('Generating index... (',p,')');
  240.     p.match(/\.extindex$/)||((p+='.extindex')&&console.warn('files index should have .extindex extension - renaming'))
  241.     if (fs.existsSync(p)){
  242.       console.error('Index already exists - aborting',p);
  243.       process.exit(12)
  244.     }
  245.    
  246.     var dirbase=fs.statSync(args['--'][i]).isDirectory()
  247.    
  248.     fs.writeFileSync(p,'');
  249.     var newIndex = {
  250.       extindex:fs.realpathSync(p),
  251.       md5index:dirbase?null:fs.realpathSync(args['--'][i]),
  252.       base:dirbase?fs.realpathSync(args['--'][i]):null,
  253.       index:dirbase?[]:String(fs.readFileSync(args['--'][i])).split('\n').filter(function(line,i){
  254.         if (!line.match(/^[0-9a-f]{32}  .+/)){
  255.           console.warn('Malformed line: ('+i+') "'+line+'"')
  256.           return false;
  257.         }
  258.         return true;
  259.       }).map(function(line){
  260.         return {
  261.           md5:line.substr(0,32),
  262.           path:line.substr(34),
  263.           stat:null
  264.         }
  265.       }).filter(function(ent){
  266.         if (!fs.existsSync(ent.path)){
  267.           console.warn('File '+ent.path+' does not exist - skipped');
  268.           return false;
  269.         }
  270.         return true;
  271.       }).map(function(ent){
  272.         ent.path=fs.realpathSync(ent.path);
  273.         return ent;
  274.       }).sort(function(a,b){
  275.         return (a.path>b.path?1:(a.path<b.path?-1:0))
  276.       }).map(function(ent,i,arr){
  277.         args.verbose&&arr.length>100000&&(i%10000==0)&&console.log('[stat:] ',(i*100/arr.length).toPrecision(3)+'%');
  278.        
  279.         ent.stat=JSON.parse(JSON.stringify(fs.statSync(ent.path)))
  280.         ent.path=ent.path.split('/').filter(function(s){return s!=''})
  281.        
  282.         return ent
  283.       })
  284.     };
  285.    
  286.     if (!newIndex.base){
  287.       var ind=newIndex.index,
  288.           longTree=ind[0].path.slice(0,-1);
  289.       for (var i = 0, l = ind.length ; i < l ; i++){
  290.         for (var j = 0, ll = ind[i].path.length-1 ; j < ll ; j++)
  291.           if (longTree[j]!=ind[i].path[j])
  292.             break;
  293.         longTree=longTree.slice(0,j);
  294.       }
  295.       newIndex.base='/'+longTree.join('/');
  296.       newIndex.index=ind.map(function(ent){
  297.         return {
  298.           stat:ent.stat,
  299.           md5:ent.md5,
  300.           path:ent.path.slice(longTree.length)
  301.         }
  302.       })
  303.     }
  304.     return newIndex;
  305.   }
  306. }).filter(function validate(index){
  307.   args.verbose&&console.log('Refreshing... (',index.extindex,')');
  308.  
  309.   //fetch current list of files
  310.  
  311.   var realindex = child_process.spawnSync('find',[index.base,'-type','f'],{cwd:'/'});
  312.   if (realindex.error||realindex.status){
  313.     console.error(realindex.error||('find exit status code: '+realindex.status))
  314.     process.exit(40);
  315.   }
  316.   var strrealindex=String(realindex.stdout);
  317.   if (strrealindex!=realindex.stdout){
  318.     console.error('Soft bug occured - it\'s serious bug and probably classifies as node bug or linux memcache bug. Should be reported');
  319.     throw ('Original string and reparsed don\'t match at '+i+' byte - system string conversion malfunction - abtorting')
  320.   }
  321.   realindex=strrealindex.split('\n').sort(function(a,b){
  322.     return (a>b?1:(a<b?-1:0))
  323.   }).filter(function(p){
  324.     if (!fs.existsSync(p)){
  325.       if (p!=''){
  326.         console.error('Something went wrong - find returned non-existing file - aborting ',p)
  327.         process.exit(41);
  328.       }
  329.       return false;
  330.     }
  331.     return true
  332.   }).map(function(p){
  333.     return p.split('/').filter(function(s){return s!=''})
  334.   });
  335.  
  336.  
  337.   var missingFile=[],
  338.       missingIndex=[];
  339.      
  340.   var diff=[];
  341.  
  342.   //find missing files
  343.  
  344.   index.index=index.index.filter(function(ent){
  345.     if (!fs.existsSync(index.base+'/'+ent.path.join('/')))
  346.       return (missingFile.push(ent)&&false);
  347.     return true;
  348.   })
  349.  
  350.   //find missing index entries
  351.  
  352.   for (i=0,j=0,l=index.index.length,ll=realindex.length; i<l&&j<ll ;){
  353.     var pi = index.base+'/'+index.index[i].path.join('/'),
  354.         pr = '/'+realindex[j].join('/');
  355.        
  356.     if (pi==pr){
  357.       i++;
  358.       j++;
  359.     } else if (pi>pr){
  360.       missingIndex.push(pr)
  361.       j++;
  362.     } else { //(pi<pr)
  363.       console.error('something went wrong - there should be no missing files at this step (possible BUG) - aborting');
  364.       process.exit(60);
  365.     }
  366.   }
  367.  
  368.   if (j!=ll)
  369.     for (;j<ll;j++)
  370.       missingIndex.push('/'+realindex[j].join('/'));
  371.  
  372.   if (i!=l)
  373.     for (;i<l;i++){
  374.       console.error('something went wrong - there should be no missing files at this step (possible BUG) - aborting');
  375.       process.exit(61);
  376.     }
  377.    
  378.  
  379.   //compare stat
  380.  
  381.   var md5queue=[];
  382.   args.verbose&&console.log('Comparing stat...');
  383.   for (var i = 0, l = index.index.length; i < l ; i++){
  384.    
  385.     args.verbose&&l>100000&&(i%10000==0)&&console.log('[stat:] ',(i*100/l).toPrecision(3)+'%');
  386.     var ent=index.index[i],
  387.         p=index.base+'/'+ent.path.join('/'),
  388.        
  389.         newStat=JSON.parse(JSON.stringify(fs.statSync(p))),
  390.         oldStat=ent.stat;
  391.        
  392.     var d={};
  393.    
  394.     var cmpKeys=['mode','uid','gid','size','mtime'];
  395.  
  396.     for (var j=0,ll=cmpKeys.length;j<ll;j++)
  397.       if (oldStat[cmpKeys[j]]!=newStat[cmpKeys[j]])
  398.         d[cmpKeys[j]]=[oldStat[cmpKeys[j]],newStat[cmpKeys[j]]];
  399.      
  400.     if (Object.keys(d).length)
  401.       diff.push({ent:ent,diff:d})
  402.      
  403.     var junkDiff=0;
  404.     if (args.careless){
  405.       ('gid' in d)&&junkDiff++;
  406.       ('uid' in d)&&junkDiff++;
  407.       ('mode' in d)&&junkDiff++;
  408.     }
  409.      
  410.     if (args.deep||
  411.         ((Object.keys(d).length>junkDiff||
  412.           ent.md5=='########### skipped ############'||
  413.           ent.md5=='############ ERROR #############')&&
  414.           !args.fast))
  415.       md5queue.push({ent:ent,diff:d});
  416.   }
  417.  
  418.   if (args.importcorrupted){
  419.     if (!fs.existsSync(args.importcorrupted))
  420.       throw 'Could not find corrupted files list';
  421.    
  422.     var list = String(fs.readFileSync(args.importcorrupted)).split('\n').filter(function(p){
  423.       return !args.md5sumcorrupted||p.match(/: FAILED$/);
  424.     }).filter(function(p){
  425.       return p.indexOf(index.base)==0
  426.     }).map(function(p){
  427.       return (args.md5sumcorrupted?p.replace(/: FAILED$/,''):p).split('/').filter(function(s){return s!=''}).slice(index.base.split('/').filter(function(s){return s!=''}).length)
  428.     })
  429.    
  430.     var files = [].concat.apply([],list.map(function(p){
  431.       var valid=index.index;
  432.       for (var i = 0, l = p.length ; i < l ; i++)
  433.         valid=valid.filter(function(ent){
  434.           return ent.path[i]==p[i];
  435.         })
  436.       return valid;
  437.     })).filter(function(e,i,arr){
  438.       return i==arr.indexOf(e)
  439.     });
  440.    
  441.     args.verbose&&console.log('Following files will be force reloaded in this index: ( '+index.base+' )\n'+files.map(function(ent){return '  '+ent.path.join('/')}).join('\n'));
  442.    
  443.     files.forEach(function(ent){
  444.       if (md5queue.filter(function(cqueue){return cqueue.ent==ent}).length==0)
  445.         md5queue.push({ent:ent,diff:{}});
  446.     })
  447.   }
  448.  
  449.   for (var i = 0, l = md5queue.length, md5queueSize=0 ; i < l ; i++)
  450.     md5queueSize+=(md5queue[i].diff.size?md5queue[i].diff.size[1]:md5queue[i].ent.stat.size)
  451.  
  452.   for (var i = 0, l = md5queue.length, md5queueProgress=0 ; i < l ; i++){
  453.     var cqueue=md5queue[i];
  454.    
  455.     args.verbose&&
  456.     (l<500||i%50==0||(cqueue.diff.size?cqueue.diff.size[1]:cqueue.ent.stat.size)>50*1024*1024)&&
  457.     console.log('[md5:] ',i+1+'/'+l,(md5queueProgress*100/md5queueSize).toPrecision(3)+'%  '+(l<500?cqueue.ent.path.join('/'):''))
  458.    
  459.     var ret=child_process.spawnSync('md5sum',[index.base+'/'+cqueue.ent.path.join('/')],{cwd:'/'});
  460.     if (ret.error||ret.status){
  461.       console.error(ret.error||'md5sum exit status code: '+ret.status+' '+String(ret.stderr));
  462.       process.exit(43);
  463.     }
  464.     ret=String(ret.stdout).replace('\n','').substr(0,32);
  465.    
  466.     if (ret!=cqueue.ent.md5){
  467.       if (Object.keys(cqueue.diff).length==0)
  468.         diff.push(cqueue);
  469.       cqueue.diff.md5=[cqueue.ent.md5,ret];
  470.     }
  471.    
  472.     md5queueProgress+=(cqueue.diff.size?cqueue.diff.size[1]:cqueue.ent.stat.size);
  473.   }
  474.  
  475.  
  476.   //apply diff
  477.  
  478.   //process.exit(0);
  479.  
  480.   diff.forEach(function(d){
  481.     var keys=Object.keys(d.diff).filter(function(k){return k!='md5'});
  482.     for (var i = 0, l = keys.length ; i < l ; i++)
  483.       d.ent.stat[keys[i]]=d.diff[keys[i]][1];
  484.     if (d.diff.md5)
  485.       d.ent.md5=d.diff.md5[1];
  486.   });
  487.  
  488.  
  489.   //add new files to index
  490.  
  491.   if (missingIndex.length){
  492.    
  493.     args.verbose&&console.log('Updating extindex... ('+missingIndex.length+' new files)')
  494.    
  495.     var baselen=index.base.split('/').filter(function(b){return b!=''}).length
  496.    
  497.     for (var i = 0, l = missingIndex.length, missingSize=0 ; i < l ; i++)
  498.       missingSize+=fs.statSync(missingIndex[i]).size;
  499.        
  500.     var missingProgress=0,l=missingIndex.length;
  501.    
  502.     missingIndex=missingIndex.map(function(p,i){
  503.       var s=fs.statSync(p);
  504.      
  505.       args.verbose&&
  506.       (l<500||i%50==0||s.size>50*1024*1024)&&
  507.       console.log(args.fast?'[stat:] ':'[md5:] ',i+1+'/'+l,(missingProgress*100/missingSize).toPrecision(3)+'%  '+(l<500?p:''));
  508.      
  509.       var ret='########### skipped ############  '+p
  510.       if (!args.fast){
  511.         ret=child_process.spawnSync('md5sum',[p],{cwd:'/'});
  512.         if (ret.error||ret.status){
  513.           console.error(ret.error||'md5sum exit status code: '+ret.status+' '+String(ret.stderr));
  514.           //process.exit(44);
  515.           ret='############ ERROR #############  '+p
  516.           errors.push([ret.status,String(ret.stderr)]);
  517.         } else
  518.           ret=String(ret.stdout).replace('\n','');
  519.       }
  520.      
  521.       missingProgress+=s.size;
  522.      
  523.       return {
  524.         stat:s,
  525.         md5:ret.substr(0,32),
  526.         path:ret.substr(34).split('/').filter(function(s){return s!=''}).slice(baselen)
  527.       }
  528.     })
  529.    
  530.     index.index=index.index.concat(missingIndex).sort(function(a,b){
  531.       var pa=a.path.join('/'),
  532.           pb=b.path.join('/');
  533.       return (pa>pb?1:(pa<pb?-1:0))
  534.     });
  535.   }
  536.  
  537.   missing.file=missing.file.concat(missingFile);
  538.   missing.index=missing.index.concat(missingIndex);
  539.   global.diff=global.diff.concat(diff);
  540.  
  541.   var now=Date.now(),
  542.       basepath=index.extindex.replace(/\/[^\/]*$/,''),
  543.       filename=index.extindex.replace(/.*\//,''),
  544.       newbase=basepath+'/extindex.old/',
  545.       newfile=newbase+filename;
  546.      
  547.      
  548.  
  549.   if ((args.forceupdate||!args.fast)&&!args.simulate){
  550.     if (!fs.existsSync(newbase))
  551.       fs.mkdirSync(newbase);
  552.    
  553.     if (missingFile.length){
  554.       console.log('Creating missing list... ('+missingFile.length+' files missing)')
  555.       fs.writeFileSync(newfile+'.'+now+'.missing',JSON.stringify(missingFile,null,2));
  556.     }
  557.    
  558.     if (missingIndex.length){
  559.       console.log('Creating new files list... ('+missingIndex.length+' new files)')
  560.       fs.writeFileSync(newfile+'.'+now+'.new',JSON.stringify(missingIndex,null,2));
  561.     }
  562.    
  563.     if (diff.length){
  564.       console.log('Creating diff list... ('+diff.length+' files different)')
  565.       fs.writeFileSync(newfile+'.'+now+'.diff',JSON.stringify(diff,null,2));
  566.     }
  567.    
  568.     if (args.forceupdate||diff.length||missingIndex.length||missingFile.length){
  569.       args.verbose&&console.log('[extindex:] ',index.extindex);
  570.      
  571.       var old = fs.readdirSync(newbase)
  572.                 .filter(function(p){return p.indexOf(filename)==0})
  573.                 .filter(function(p){return p.match(/\.old\.[0-9]+$/)});
  574.       old.filter(function(p){return p.match(/[0-9]+$/)[0]>=args.backup}).forEach(function(p){fs.unlinkSync(newbase+p)});
  575.      
  576.       if (args.backup){
  577.         for (var b = args.backup-1 ; b > 0 ; b--)
  578.           old.filter(function(p){return p.match(/[0-9]+$/)[0]==b}).forEach(function(p){fs.renameSync(newbase+p,newbase+p.replace(/[0-9]+$/,b+1))})
  579.         fs.renameSync(index.extindex,newfile+'.old.1');
  580.         for (var i = 0, l = index.subindexnum ; i < l ; i++)
  581.           fs.renameSync(index.extindex+'.part'+i,newfile+'.part'+i+'.old.1');
  582.       }
  583.      
  584.       subindexes=[];
  585.       while (index.index.length>25000)
  586.         subindexes.push(index.index.splice(0,25000));
  587.       subindexes.push(index.index);
  588.       if(subindexes.length>1){
  589.         index.index=null;
  590.         index.subindexnum=subindexes.length;
  591.       } else {
  592.         subindexes=[];
  593.         index.subindexnum=0;
  594.       }
  595.       var filenames=[index.extindex],
  596.           filecontents=[index];
  597.          
  598.       for (var i = 0, l = subindexes.length ; i < l ; i++){
  599.         filenames.push(index.extindex+'.part'+i);
  600.         filecontents.push(subindexes[i]);
  601.       }
  602.      
  603.       for (var i = 0, l = filenames.length ; i < l ; i++){
  604.         args.verbose&&console.log('[extindex:] part '+(i+1)+'/'+l);
  605.      
  606.         var content = JSON.stringify(filecontents[i],null,2),
  607.             sha512;
  608.        
  609.         var sha = child_process.spawnSync('sha512sum',[],{input:content})
  610.         if (sha.error||sha.status){
  611.           console.error(sha.error||'sha512sum exit status code: '+sha.status+' '+String(sha.stderr));
  612.           process.exit(45);
  613.         }
  614.         sha512 = String(sha.stdout).substr(0,128);
  615.        
  616.         try {
  617.           fs.writeFileSync(filenames[i],sha512+content);
  618.           var file = fs.readFileSync(filenames[i]);
  619.           if (file!=(sha512+content))
  620.             throw ':c';
  621.         } catch (e) {
  622.           console.error('Error occured during index write - retry')
  623.           fs.writeFileSync(filenames[i],sha512+content);
  624.           var file = fs.readFileSync(filenames[i]);
  625.           if (file!=(sha512+content))
  626.             throw 'Could not properly write index file - operation totally failed, all your data is lost and ur doomed :< - aborting';
  627.         }
  628.       }
  629.       if (subindexes.length)
  630.         index.index=[].concat.apply([],subindexes)
  631.      
  632.     }
  633.    
  634.     if (args.export){
  635.       var md5indexpath=basepath+'/md5index/'+filename.replace(/\.extindex$/,'.md5index');
  636.       args.verbose&&console.log('[md5index:] ',md5indexpath)
  637.      
  638.       if (!fs.existsSync(basepath+'/md5index'))
  639.         fs.mkdirSync(basepath+'/md5index');
  640.      
  641.       if (fs.existsSync(md5indexpath+'.old'))
  642.         fs.unlinkSync(md5indexpath+'.old');
  643.       if (fs.existsSync(md5indexpath))
  644.         fs.renameSync(md5indexpath,md5indexpath+'.old');
  645.      
  646.       fs.writeFileSync(md5indexpath,index.index.map(function(ent){
  647.         return ent.md5+'  '+index.base+'/'+ent.path.join('/');
  648.       }).join('\n'))
  649.     }
  650.   } else
  651.     args.verbose&&console.log((args.fast?'Fast':'Simulate')+' mode - log write skipped');
  652.  
  653.   return true;
  654. });
  655.  
  656.  
  657. Object.defineProperties(global,{
  658.   shortdiff:{
  659.     get:function(){
  660.       return diff.map(function(d){
  661.         return Object.keys(d.diff).sort().join(',') + ' -- ' + d.ent.path.join('/');
  662.       })
  663.     }
  664.   },
  665.  
  666.   shortmissing:{
  667.     get:function(){
  668.       return missing.file.map(function(ent){
  669.         return ent.path.join('/');
  670.       })
  671.     }
  672.   },
  673.  
  674.   shortnew:{
  675.     get:function(){
  676.       return missing.index.map(function(ent){
  677.         return ent.path.join('/');
  678.       })
  679.     }
  680.   }
  681. });
  682.  
  683.  
  684. if (args.interactive||(args.halfinteractive&&missing.file.length+missing.index.length+diff.length>0))
  685.   require('/usr/lib/node_modules/interactive/').start(this);
  686. else
  687.   args.verbose&&console.log('Nothing to do - exiting');
  688.  
  689.  
  690. function buildExtendedTree(index){
  691.   var extIndex={
  692.         md5index:index.md5index,
  693.         basePath:index.base,
  694.         tree:null
  695.       },
  696.       tree={};
  697.  
  698.   (function buildDirTree(ind,tree){
  699.    
  700.     for (var i = 0, l = ind.length ; i<l ; i++){
  701.      
  702.     }
  703.    
  704.   })(index.index,tree);
  705.      
  706.   extIndex.tree=tree;
  707.   return extIndex;
  708. }
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy. OK, I Understand
 
Top