Guest User

Untitled

a guest
Jul 24th, 2016
1,703
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. #!/usr/bin/node
  2.  
  3. //
  4.  
  5. //mock file index design
  6. ({
  7. stat:{},
  8. '/':{}
  9. })
  10.  
  11.  
  12.  
  13.  
  14.  
  15.  
  16.  
  17. //
  18.  
  19. fs=require('fs');
  20. child_process=require('child_process');
  21.  
  22. var argsRules = {
  23. '--':true,
  24. boolean:[
  25. 'help',
  26. 'verbose',
  27. 'deep',
  28. 'fast',
  29. 'careless',
  30. 'interactive',
  31. 'halfinteractive',
  32. 'all',
  33. 'simulate',
  34. 'export',
  35. 'init',
  36. 'md5sumcorrupted',
  37. 'checkindex',
  38. 'forceupdate',
  39. 'legacyconvert',
  40. 'backup'
  41. ],
  42. string:[
  43. 'mountpoint',
  44. 'indexdir',
  45. 'importcorrupted'
  46. ],
  47. alias:{
  48. help:['h'],
  49. verbose:['v'],
  50. deep:['d'],
  51. fast:['f'],
  52. careless:['L'],
  53. interactive:['i'],
  54. halfinteractive:['I'],
  55. all:['a'],
  56. simulate:['s'],
  57. export:['e'],
  58. importcorrupted:['import'],
  59. init:['g'],
  60. backup:['b'],
  61. mountpoint:['mnt','M']
  62. },
  63. description:{
  64. help:'show this help',
  65. verbose:'be verbose',
  66. deep:'force md5 recheck',
  67. fast:'force md5 skip',
  68. careless:'don\'t recheck md5 if only less important stat values changed (uid,gid,permissions)',
  69. interactive:'enable interactive mode',
  70. halfinteractive:'enable interactive mode only if index changed',
  71. mountpoint:'specify default drives mountpoint',
  72. indexdir:'specify default index dir prefix',
  73. all:'scan default drives mount dir searching for all available indexes',
  74. simulate:'enable simulation mode (do modify/write any files)',
  75. export:'export md5 hashes in md5sum -c readable format for faster validation than --deep',
  76. importcorrupted:'import list of files for force md5 validation',
  77. md5sumcorrupted:'read importcorrupted file as md5sum -c output',
  78. init:'create new index from scratch',
  79. checkindex:'Validate index sha512 before parsing',
  80. forceupdate:'update extindex file even if there were no changes noted',
  81. legacyconvert:'read file in legacy mode (do not expect first 128 bytes to be sha512)',
  82. backup:'number of backup copies to keep'
  83. },
  84. default:{
  85. verbose:true,
  86. halfinteractive:true,
  87. mountpoint:'/dmnt',
  88. indexdir:'/etc/md5index',
  89. checkindex:true,
  90. backup:3
  91. }
  92. }
  93.  
  94. args=require('/usr/lib/node_modules/minimist/')(process.argv.slice(2),argsRules);
  95.  
  96. (function(){
  97. var validArgs=[].concat.apply(['--','_'],argsRules.boolean.concat(argsRules.string).map(function(key){return [key].concat((key in argsRules.alias)?argsRules.alias[key]:[])})),
  98. invalidArgs=[];
  99. Object.keys(args).forEach(function(key){validArgs.indexOf(key)==-1&&invalidArgs.push(key)})
  100. if (invalidArgs.length){
  101. console.error('invalidArgs: \n'+invalidArgs.join('\n'));
  102. process.exit(7);
  103. }
  104. })()
  105.  
  106. if (args.help){
  107. var keys = Object.keys(argsRules.description),white = '',
  108. args = Object.keys(argsRules.description).map(function(arg,i,arr){return ' '+((arg.length>1?'--':'-')+arg+', '+(argsRules.alias[arg]?(argsRules.alias[arg].map(function(a){return (a.length>1?'--':'-')+a}).join(', ')):''))+' '});
  109. args.forEach(function(a){if (a.length>white.length) white=a.split('').map(function(){return ' '}).join('')})
  110. console.log('\n'+args.map(function(a,i){return a+white.substr(a.length)+argsRules.description[keys[i]]+((keys[i] in argsRules.default)?(' [DEFAULT:'+argsRules.default[keys[i]]+']'):'')}).join('\n'));
  111. process.exit(0);
  112. }
  113.  
  114. if (args.init)
  115. args.forceupdate=true;
  116.  
  117. if (args.legacyconvert){
  118. args.checkindex=false,
  119. args.forceupdate=true;
  120. }
  121.  
  122. errors=[];
  123.  
  124. global.missing={
  125. file:[],
  126. index:[]
  127. }
  128.  
  129. global.diff=[]
  130.  
  131. indexes=(args.all?
  132. (!fs.existsSync(args.mountpoint)?(function(){throw 'Could not find default drives location'})():
  133. [].concat.apply([],fs.readdirSync(args.mountpoint).map(function(p){
  134. return args.mountpoint+'/'+p+'/'+args.indexdir;
  135. }).filter(function(p){
  136. return fs.existsSync(p)&&fs.statSync(p).isDirectory();
  137. }).map(function(p){
  138. return fs.readdirSync(p).filter(function(f){
  139. return f.match(/.*\.extindex$/)
  140. }).map(function(f){
  141. console.log('Found: ',p+'/'+f);
  142. return p+'/'+f
  143. })
  144. }))):
  145. args._).filter(function(p,i){
  146. if (args.init^(!fs.existsSync(p))){
  147. console.error('Index '+(args.init?'already exists':'does not exist')+' - aborting',p);
  148. process.exit(10);
  149. }
  150. if (args.init&&!fs.existsSync(args['--'][i])){
  151. console.error('Could not find base dir / source md5index for '+p)
  152. process.exit(13);
  153. }
  154. return true;
  155. }).map(function(p,i){
  156. if (!args.init){
  157. args.verbose&&console.log('Reading index... (',p,')');
  158. var bfile = fs.readFileSync(fs.realpathSync(p)),
  159. sha512 = bfile.slice(0,128),
  160. content = bfile.slice(128);
  161.  
  162. var sha = child_process.spawnSync('sha512sum',[],{input:content})
  163. if (sha.error||sha.status){
  164. console.error(sha.error||'sha512sum exit status code: '+sha.status+' '+String(sha.stderr));
  165. process.exit(45);
  166. }
  167.  
  168. if (String(sha.stdout).substr(0,128)!=sha512){
  169. if (args.checkindex){
  170. console.error('Index sha512 check failed - index is probably damaged - aborting\n',sha512,'\n',String(sha.stdout).substr(0,128))
  171. process.exit(14);
  172. } else
  173. console.warn('Index sha512sum check failed - index is probably damaged - forced ignore\n',sha512,'\n',String(sha.stdout).substr(0,128));
  174. }
  175. try{
  176. var pind = JSON.parse(args.legacyconvert?bfile:content),
  177. ostr = String(args.legacyconvert?bfile:content),
  178. str = JSON.stringify(pind,null,2);
  179.  
  180. for (var i = 0, l = str.length ; i < l ; i++)
  181. if (str[i]!=ostr[i]){
  182. console.error('Soft bug occured - it\'s serious bug and probably classifies as node bug or linux memcache bug. Should be reported');
  183. throw ('Original string and reparsed don\'t match at '+i+' byte - system string conversion malfunction - abtorting')
  184. }
  185.  
  186. if (!(typeof pind == 'object' && !Array.isArray(pind) && 'index' in pind))
  187. throw 'Invalid index file';
  188.  
  189. var curpath = fs.realpathSync(p);
  190. if (pind.extindex!=curpath)
  191. console.warn('Index has been moved from its previous location - it may result in errors');
  192. pind.extindex=curpath;
  193.  
  194. if (!pind.index){
  195. pind.index=[];
  196. for (var i = 0, l = pind.subindexnum ; i < l ; ++i)
  197. pind.index.push(pind.extindex+'.part'+i);
  198.  
  199. pind.index=[].concat.apply([],pind.index.map(function(p,i){
  200. args.verbose&&console.log('Reading subindex... (',p,')');
  201. if (!fs.existsSync(p))
  202. throw 'Expected subindex does not exist';
  203.  
  204. var bfile = fs.readFileSync(p),
  205. sha512 = bfile.slice(0,128),
  206. content = bfile.slice(128);
  207.  
  208. var sha = child_process.spawnSync('sha512sum',[],{input:content});
  209. if(sha.error||sha.status)
  210. throw sha.error||'sha512sum exit status code for subindex '+i+': '+sha.status+' '+String(sha.stderr);
  211. if((sha=String(sha.stdout).substr(0,128))!=sha512){
  212. if (args.checkindex)
  213. throw 'Subindex sha512 check failed - subindex is probably damaged - aborting\n'+sha512+'\n'+sha;
  214. else
  215. console.warn('Subindex sha512 check failed - subindex is probably damaged - forced ignore\n'+sha512+'\n'+sha);
  216. }
  217.  
  218. var sind = JSON.parse(content),
  219. ostr=String(content),
  220. str = JSON.stringify(sind,null,2);
  221.  
  222. for (var i = 0, l = str.length; i < l ; i++)
  223. if (str[i]!=ostr[i]){
  224. console.error('Soft bug occured - it\'s serious bug and probably classifies as node bug or linux memcache bug. Should be reported');
  225. throw ('Original string and reparsed don\'t match at '+i+' byte - system string conversion malfunction - abtorting')
  226. }
  227.  
  228. return sind;
  229.  
  230. }));
  231. }
  232.  
  233. return pind;
  234. } catch (e) {
  235. console.error('Could not read index - aborting',p,e);
  236. process.exit(11);
  237. }
  238. } else {
  239. args.verbose&&console.log('Generating index... (',p,')');
  240. p.match(/\.extindex$/)||((p+='.extindex')&&console.warn('files index should have .extindex extension - renaming'))
  241. if (fs.existsSync(p)){
  242. console.error('Index already exists - aborting',p);
  243. process.exit(12)
  244. }
  245.  
  246. var dirbase=fs.statSync(args['--'][i]).isDirectory()
  247.  
  248. fs.writeFileSync(p,'');
  249. var newIndex = {
  250. extindex:fs.realpathSync(p),
  251. md5index:dirbase?null:fs.realpathSync(args['--'][i]),
  252. base:dirbase?fs.realpathSync(args['--'][i]):null,
  253. index:dirbase?[]:String(fs.readFileSync(args['--'][i])).split('\n').filter(function(line,i){
  254. if (!line.match(/^[0-9a-f]{32} .+/)){
  255. console.warn('Malformed line: ('+i+') "'+line+'"')
  256. return false;
  257. }
  258. return true;
  259. }).map(function(line){
  260. return {
  261. md5:line.substr(0,32),
  262. path:line.substr(34),
  263. stat:null
  264. }
  265. }).filter(function(ent){
  266. if (!fs.existsSync(ent.path)){
  267. console.warn('File '+ent.path+' does not exist - skipped');
  268. return false;
  269. }
  270. return true;
  271. }).map(function(ent){
  272. ent.path=fs.realpathSync(ent.path);
  273. return ent;
  274. }).sort(function(a,b){
  275. return (a.path>b.path?1:(a.path<b.path?-1:0))
  276. }).map(function(ent,i,arr){
  277. args.verbose&&arr.length>100000&&(i%10000==0)&&console.log('[stat:] ',(i*100/arr.length).toPrecision(3)+'%');
  278.  
  279. ent.stat=JSON.parse(JSON.stringify(fs.statSync(ent.path)))
  280. ent.path=ent.path.split('/').filter(function(s){return s!=''})
  281.  
  282. return ent
  283. })
  284. };
  285.  
  286. if (!newIndex.base){
  287. var ind=newIndex.index,
  288. longTree=ind[0].path.slice(0,-1);
  289. for (var i = 0, l = ind.length ; i < l ; i++){
  290. for (var j = 0, ll = ind[i].path.length-1 ; j < ll ; j++)
  291. if (longTree[j]!=ind[i].path[j])
  292. break;
  293. longTree=longTree.slice(0,j);
  294. }
  295. newIndex.base='/'+longTree.join('/');
  296. newIndex.index=ind.map(function(ent){
  297. return {
  298. stat:ent.stat,
  299. md5:ent.md5,
  300. path:ent.path.slice(longTree.length)
  301. }
  302. })
  303. }
  304. return newIndex;
  305. }
  306. }).filter(function validate(index){
  307. args.verbose&&console.log('Refreshing... (',index.extindex,')');
  308.  
  309. //fetch current list of files
  310.  
  311. var realindex = child_process.spawnSync('find',[index.base,'-type','f'],{cwd:'/'});
  312. if (realindex.error||realindex.status){
  313. console.error(realindex.error||('find exit status code: '+realindex.status))
  314. process.exit(40);
  315. }
  316. var strrealindex=String(realindex.stdout);
  317. if (strrealindex!=realindex.stdout){
  318. console.error('Soft bug occured - it\'s serious bug and probably classifies as node bug or linux memcache bug. Should be reported');
  319. throw ('Original string and reparsed don\'t match at '+i+' byte - system string conversion malfunction - abtorting')
  320. }
  321. realindex=strrealindex.split('\n').sort(function(a,b){
  322. return (a>b?1:(a<b?-1:0))
  323. }).filter(function(p){
  324. if (!fs.existsSync(p)){
  325. if (p!=''){
  326. console.error('Something went wrong - find returned non-existing file - aborting ',p)
  327. process.exit(41);
  328. }
  329. return false;
  330. }
  331. return true
  332. }).map(function(p){
  333. return p.split('/').filter(function(s){return s!=''})
  334. });
  335.  
  336.  
  337. var missingFile=[],
  338. missingIndex=[];
  339.  
  340. var diff=[];
  341.  
  342. //find missing files
  343.  
  344. index.index=index.index.filter(function(ent){
  345. if (!fs.existsSync(index.base+'/'+ent.path.join('/')))
  346. return (missingFile.push(ent)&&false);
  347. return true;
  348. })
  349.  
  350. //find missing index entries
  351.  
  352. for (i=0,j=0,l=index.index.length,ll=realindex.length; i<l&&j<ll ;){
  353. var pi = index.base+'/'+index.index[i].path.join('/'),
  354. pr = '/'+realindex[j].join('/');
  355.  
  356. if (pi==pr){
  357. i++;
  358. j++;
  359. } else if (pi>pr){
  360. missingIndex.push(pr)
  361. j++;
  362. } else { //(pi<pr)
  363. console.error('something went wrong - there should be no missing files at this step (possible BUG) - aborting');
  364. process.exit(60);
  365. }
  366. }
  367.  
  368. if (j!=ll)
  369. for (;j<ll;j++)
  370. missingIndex.push('/'+realindex[j].join('/'));
  371.  
  372. if (i!=l)
  373. for (;i<l;i++){
  374. console.error('something went wrong - there should be no missing files at this step (possible BUG) - aborting');
  375. process.exit(61);
  376. }
  377.  
  378.  
  379. //compare stat
  380.  
  381. var md5queue=[];
  382. args.verbose&&console.log('Comparing stat...');
  383. for (var i = 0, l = index.index.length; i < l ; i++){
  384.  
  385. args.verbose&&l>100000&&(i%10000==0)&&console.log('[stat:] ',(i*100/l).toPrecision(3)+'%');
  386. var ent=index.index[i],
  387. p=index.base+'/'+ent.path.join('/'),
  388.  
  389. newStat=JSON.parse(JSON.stringify(fs.statSync(p))),
  390. oldStat=ent.stat;
  391.  
  392. var d={};
  393.  
  394. var cmpKeys=['mode','uid','gid','size','mtime'];
  395.  
  396. for (var j=0,ll=cmpKeys.length;j<ll;j++)
  397. if (oldStat[cmpKeys[j]]!=newStat[cmpKeys[j]])
  398. d[cmpKeys[j]]=[oldStat[cmpKeys[j]],newStat[cmpKeys[j]]];
  399.  
  400. if (Object.keys(d).length)
  401. diff.push({ent:ent,diff:d})
  402.  
  403. var junkDiff=0;
  404. if (args.careless){
  405. ('gid' in d)&&junkDiff++;
  406. ('uid' in d)&&junkDiff++;
  407. ('mode' in d)&&junkDiff++;
  408. }
  409.  
  410. if (args.deep||
  411. ((Object.keys(d).length>junkDiff||
  412. ent.md5=='########### skipped ############'||
  413. ent.md5=='############ ERROR #############')&&
  414. !args.fast))
  415. md5queue.push({ent:ent,diff:d});
  416. }
  417.  
  418. if (args.importcorrupted){
  419. if (!fs.existsSync(args.importcorrupted))
  420. throw 'Could not find corrupted files list';
  421.  
  422. var list = String(fs.readFileSync(args.importcorrupted)).split('\n').filter(function(p){
  423. return !args.md5sumcorrupted||p.match(/: FAILED$/);
  424. }).filter(function(p){
  425. return p.indexOf(index.base)==0
  426. }).map(function(p){
  427. return (args.md5sumcorrupted?p.replace(/: FAILED$/,''):p).split('/').filter(function(s){return s!=''}).slice(index.base.split('/').filter(function(s){return s!=''}).length)
  428. })
  429.  
  430. var files = [].concat.apply([],list.map(function(p){
  431. var valid=index.index;
  432. for (var i = 0, l = p.length ; i < l ; i++)
  433. valid=valid.filter(function(ent){
  434. return ent.path[i]==p[i];
  435. })
  436. return valid;
  437. })).filter(function(e,i,arr){
  438. return i==arr.indexOf(e)
  439. });
  440.  
  441. args.verbose&&console.log('Following files will be force reloaded in this index: ( '+index.base+' )\n'+files.map(function(ent){return ' '+ent.path.join('/')}).join('\n'));
  442.  
  443. files.forEach(function(ent){
  444. if (md5queue.filter(function(cqueue){return cqueue.ent==ent}).length==0)
  445. md5queue.push({ent:ent,diff:{}});
  446. })
  447. }
  448.  
  449. for (var i = 0, l = md5queue.length, md5queueSize=0 ; i < l ; i++)
  450. md5queueSize+=(md5queue[i].diff.size?md5queue[i].diff.size[1]:md5queue[i].ent.stat.size)
  451.  
  452. for (var i = 0, l = md5queue.length, md5queueProgress=0 ; i < l ; i++){
  453. var cqueue=md5queue[i];
  454.  
  455. args.verbose&&
  456. (l<500||i%50==0||(cqueue.diff.size?cqueue.diff.size[1]:cqueue.ent.stat.size)>50*1024*1024)&&
  457. console.log('[md5:] ',i+1+'/'+l,(md5queueProgress*100/md5queueSize).toPrecision(3)+'% '+(l<500?cqueue.ent.path.join('/'):''))
  458.  
  459. var ret=child_process.spawnSync('md5sum',[index.base+'/'+cqueue.ent.path.join('/')],{cwd:'/'});
  460. if (ret.error||ret.status){
  461. console.error(ret.error||'md5sum exit status code: '+ret.status+' '+String(ret.stderr));
  462. process.exit(43);
  463. }
  464. ret=String(ret.stdout).replace('\n','').substr(0,32);
  465.  
  466. if (ret!=cqueue.ent.md5){
  467. if (Object.keys(cqueue.diff).length==0)
  468. diff.push(cqueue);
  469. cqueue.diff.md5=[cqueue.ent.md5,ret];
  470. }
  471.  
  472. md5queueProgress+=(cqueue.diff.size?cqueue.diff.size[1]:cqueue.ent.stat.size);
  473. }
  474.  
  475.  
  476. //apply diff
  477.  
  478. //process.exit(0);
  479.  
  480. diff.forEach(function(d){
  481. var keys=Object.keys(d.diff).filter(function(k){return k!='md5'});
  482. for (var i = 0, l = keys.length ; i < l ; i++)
  483. d.ent.stat[keys[i]]=d.diff[keys[i]][1];
  484. if (d.diff.md5)
  485. d.ent.md5=d.diff.md5[1];
  486. });
  487.  
  488.  
  489. //add new files to index
  490.  
  491. if (missingIndex.length){
  492.  
  493. args.verbose&&console.log('Updating extindex... ('+missingIndex.length+' new files)')
  494.  
  495. var baselen=index.base.split('/').filter(function(b){return b!=''}).length
  496.  
  497. for (var i = 0, l = missingIndex.length, missingSize=0 ; i < l ; i++)
  498. missingSize+=fs.statSync(missingIndex[i]).size;
  499.  
  500. var missingProgress=0,l=missingIndex.length;
  501.  
  502. missingIndex=missingIndex.map(function(p,i){
  503. var s=fs.statSync(p);
  504.  
  505. args.verbose&&
  506. (l<500||i%50==0||s.size>50*1024*1024)&&
  507. console.log(args.fast?'[stat:] ':'[md5:] ',i+1+'/'+l,(missingProgress*100/missingSize).toPrecision(3)+'% '+(l<500?p:''));
  508.  
  509. var ret='########### skipped ############ '+p
  510. if (!args.fast){
  511. ret=child_process.spawnSync('md5sum',[p],{cwd:'/'});
  512. if (ret.error||ret.status){
  513. console.error(ret.error||'md5sum exit status code: '+ret.status+' '+String(ret.stderr));
  514. //process.exit(44);
  515. ret='############ ERROR ############# '+p
  516. errors.push([ret.status,String(ret.stderr)]);
  517. } else
  518. ret=String(ret.stdout).replace('\n','');
  519. }
  520.  
  521. missingProgress+=s.size;
  522.  
  523. return {
  524. stat:s,
  525. md5:ret.substr(0,32),
  526. path:ret.substr(34).split('/').filter(function(s){return s!=''}).slice(baselen)
  527. }
  528. })
  529.  
  530. index.index=index.index.concat(missingIndex).sort(function(a,b){
  531. var pa=a.path.join('/'),
  532. pb=b.path.join('/');
  533. return (pa>pb?1:(pa<pb?-1:0))
  534. });
  535. }
  536.  
  537. missing.file=missing.file.concat(missingFile);
  538. missing.index=missing.index.concat(missingIndex);
  539. global.diff=global.diff.concat(diff);
  540.  
  541. var now=Date.now(),
  542. basepath=index.extindex.replace(/\/[^\/]*$/,''),
  543. filename=index.extindex.replace(/.*\//,''),
  544. newbase=basepath+'/extindex.old/',
  545. newfile=newbase+filename;
  546.  
  547.  
  548.  
  549. if ((args.forceupdate||!args.fast)&&!args.simulate){
  550. if (!fs.existsSync(newbase))
  551. fs.mkdirSync(newbase);
  552.  
  553. if (missingFile.length){
  554. console.log('Creating missing list... ('+missingFile.length+' files missing)')
  555. fs.writeFileSync(newfile+'.'+now+'.missing',JSON.stringify(missingFile,null,2));
  556. }
  557.  
  558. if (missingIndex.length){
  559. console.log('Creating new files list... ('+missingIndex.length+' new files)')
  560. fs.writeFileSync(newfile+'.'+now+'.new',JSON.stringify(missingIndex,null,2));
  561. }
  562.  
  563. if (diff.length){
  564. console.log('Creating diff list... ('+diff.length+' files different)')
  565. fs.writeFileSync(newfile+'.'+now+'.diff',JSON.stringify(diff,null,2));
  566. }
  567.  
  568. if (args.forceupdate||diff.length||missingIndex.length||missingFile.length){
  569. args.verbose&&console.log('[extindex:] ',index.extindex);
  570.  
  571. var old = fs.readdirSync(newbase)
  572. .filter(function(p){return p.indexOf(filename)==0})
  573. .filter(function(p){return p.match(/\.old\.[0-9]+$/)});
  574. old.filter(function(p){return p.match(/[0-9]+$/)[0]>=args.backup}).forEach(function(p){fs.unlinkSync(newbase+p)});
  575.  
  576. if (args.backup){
  577. for (var b = args.backup-1 ; b > 0 ; b--)
  578. old.filter(function(p){return p.match(/[0-9]+$/)[0]==b}).forEach(function(p){fs.renameSync(newbase+p,newbase+p.replace(/[0-9]+$/,b+1))})
  579. fs.renameSync(index.extindex,newfile+'.old.1');
  580. for (var i = 0, l = index.subindexnum ; i < l ; i++)
  581. fs.renameSync(index.extindex+'.part'+i,newfile+'.part'+i+'.old.1');
  582. }
  583.  
  584. subindexes=[];
  585. while (index.index.length>25000)
  586. subindexes.push(index.index.splice(0,25000));
  587. subindexes.push(index.index);
  588. if(subindexes.length>1){
  589. index.index=null;
  590. index.subindexnum=subindexes.length;
  591. } else {
  592. subindexes=[];
  593. index.subindexnum=0;
  594. }
  595. var filenames=[index.extindex],
  596. filecontents=[index];
  597.  
  598. for (var i = 0, l = subindexes.length ; i < l ; i++){
  599. filenames.push(index.extindex+'.part'+i);
  600. filecontents.push(subindexes[i]);
  601. }
  602.  
  603. for (var i = 0, l = filenames.length ; i < l ; i++){
  604. args.verbose&&console.log('[extindex:] part '+(i+1)+'/'+l);
  605.  
  606. var content = JSON.stringify(filecontents[i],null,2),
  607. sha512;
  608.  
  609. var sha = child_process.spawnSync('sha512sum',[],{input:content})
  610. if (sha.error||sha.status){
  611. console.error(sha.error||'sha512sum exit status code: '+sha.status+' '+String(sha.stderr));
  612. process.exit(45);
  613. }
  614. sha512 = String(sha.stdout).substr(0,128);
  615.  
  616. try {
  617. fs.writeFileSync(filenames[i],sha512+content);
  618. var file = fs.readFileSync(filenames[i]);
  619. if (file!=(sha512+content))
  620. throw ':c';
  621. } catch (e) {
  622. console.error('Error occured during index write - retry')
  623. fs.writeFileSync(filenames[i],sha512+content);
  624. var file = fs.readFileSync(filenames[i]);
  625. if (file!=(sha512+content))
  626. throw 'Could not properly write index file - operation totally failed, all your data is lost and ur doomed :< - aborting';
  627. }
  628. }
  629. if (subindexes.length)
  630. index.index=[].concat.apply([],subindexes)
  631.  
  632. }
  633.  
  634. if (args.export){
  635. var md5indexpath=basepath+'/md5index/'+filename.replace(/\.extindex$/,'.md5index');
  636. args.verbose&&console.log('[md5index:] ',md5indexpath)
  637.  
  638. if (!fs.existsSync(basepath+'/md5index'))
  639. fs.mkdirSync(basepath+'/md5index');
  640.  
  641. if (fs.existsSync(md5indexpath+'.old'))
  642. fs.unlinkSync(md5indexpath+'.old');
  643. if (fs.existsSync(md5indexpath))
  644. fs.renameSync(md5indexpath,md5indexpath+'.old');
  645.  
  646. fs.writeFileSync(md5indexpath,index.index.map(function(ent){
  647. return ent.md5+' '+index.base+'/'+ent.path.join('/');
  648. }).join('\n'))
  649. }
  650. } else
  651. args.verbose&&console.log((args.fast?'Fast':'Simulate')+' mode - log write skipped');
  652.  
  653. return true;
  654. });
  655.  
  656.  
  657. Object.defineProperties(global,{
  658. shortdiff:{
  659. get:function(){
  660. return diff.map(function(d){
  661. return Object.keys(d.diff).sort().join(',') + ' -- ' + d.ent.path.join('/');
  662. })
  663. }
  664. },
  665.  
  666. shortmissing:{
  667. get:function(){
  668. return missing.file.map(function(ent){
  669. return ent.path.join('/');
  670. })
  671. }
  672. },
  673.  
  674. shortnew:{
  675. get:function(){
  676. return missing.index.map(function(ent){
  677. return ent.path.join('/');
  678. })
  679. }
  680. }
  681. });
  682.  
  683.  
  684. if (args.interactive||(args.halfinteractive&&missing.file.length+missing.index.length+diff.length>0))
  685. require('/usr/lib/node_modules/interactive/').start(this);
  686. else
  687. args.verbose&&console.log('Nothing to do - exiting');
  688.  
  689.  
  690. function buildExtendedTree(index){
  691. var extIndex={
  692. md5index:index.md5index,
  693. basePath:index.base,
  694. tree:null
  695. },
  696. tree={};
  697.  
  698. (function buildDirTree(ind,tree){
  699.  
  700. for (var i = 0, l = ind.length ; i<l ; i++){
  701.  
  702. }
  703.  
  704. })(index.index,tree);
  705.  
  706. extIndex.tree=tree;
  707. return extIndex;
  708. }
RAW Paste Data