API tools faq
paste
Advertisement
Guest User

Untitled

a guest
Jul 24th, 2016
6,417
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 23.50 KB | None | 0 0
raw download clone embed print report
  1. #!/usr/bin/node
  2.  
  3. //
  4.  
  5. //mock file index design
  6. ({
  7. stat:{},
  8. '/':{}
  9. })
  10.  
  11.  
  12.  
  13.  
  14.  
  15.  
  16.  
  17. //
  18.  
  19. fs=require('fs');
  20. child_process=require('child_process');
  21.  
  22. var argsRules = {
  23. '--':true,
  24. boolean:[
  25. 'help',
  26. 'verbose',
  27. 'deep',
  28. 'fast',
  29. 'careless',
  30. 'interactive',
  31. 'halfinteractive',
  32. 'all',
  33. 'simulate',
  34. 'export',
  35. 'init',
  36. 'md5sumcorrupted',
  37. 'checkindex',
  38. 'forceupdate',
  39. 'legacyconvert',
  40. 'backup'
  41. ],
  42. string:[
  43. 'mountpoint',
  44. 'indexdir',
  45. 'importcorrupted'
  46. ],
  47. alias:{
  48. help:['h'],
  49. verbose:['v'],
  50. deep:['d'],
  51. fast:['f'],
  52. careless:['L'],
  53. interactive:['i'],
  54. halfinteractive:['I'],
  55. all:['a'],
  56. simulate:['s'],
  57. export:['e'],
  58. importcorrupted:['import'],
  59. init:['g'],
  60. backup:['b'],
  61. mountpoint:['mnt','M']
  62. },
  63. description:{
  64. help:'show this help',
  65. verbose:'be verbose',
  66. deep:'force md5 recheck',
  67. fast:'force md5 skip',
  68. careless:'don\'t recheck md5 if only less important stat values changed (uid,gid,permissions)',
  69. interactive:'enable interactive mode',
  70. halfinteractive:'enable interactive mode only if index changed',
  71. mountpoint:'specify default drives mountpoint',
  72. indexdir:'specify default index dir prefix',
  73. all:'scan default drives mount dir searching for all available indexes',
  74. simulate:'enable simulation mode (do modify/write any files)',
  75. export:'export md5 hashes in md5sum -c readable format for faster validation than --deep',
  76. importcorrupted:'import list of files for force md5 validation',
  77. md5sumcorrupted:'read importcorrupted file as md5sum -c output',
  78. init:'create new index from scratch',
  79. checkindex:'Validate index sha512 before parsing',
  80. forceupdate:'update extindex file even if there were no changes noted',
  81. legacyconvert:'read file in legacy mode (do not expect first 128 bytes to be sha512)',
  82. backup:'number of backup copies to keep'
  83. },
  84. default:{
  85. verbose:true,
  86. halfinteractive:true,
  87. mountpoint:'/dmnt',
  88. indexdir:'/etc/md5index',
  89. checkindex:true,
  90. backup:3
  91. }
  92. }
  93.  
  94. args=require('/usr/lib/node_modules/minimist/')(process.argv.slice(2),argsRules);
  95.  
  96. (function(){
  97. var validArgs=[].concat.apply(['--','_'],argsRules.boolean.concat(argsRules.string).map(function(key){return [key].concat((key in argsRules.alias)?argsRules.alias[key]:[])})),
  98. invalidArgs=[];
  99. Object.keys(args).forEach(function(key){validArgs.indexOf(key)==-1&&invalidArgs.push(key)})
  100. if (invalidArgs.length){
  101. console.error('invalidArgs: \n'+invalidArgs.join('\n'));
  102. process.exit(7);
  103. }
  104. })()
  105.  
  106. if (args.help){
  107. var keys = Object.keys(argsRules.description),white = '',
  108. args = Object.keys(argsRules.description).map(function(arg,i,arr){return ' '+((arg.length>1?'--':'-')+arg+', '+(argsRules.alias[arg]?(argsRules.alias[arg].map(function(a){return (a.length>1?'--':'-')+a}).join(', ')):''))+' '});
  109. args.forEach(function(a){if (a.length>white.length) white=a.split('').map(function(){return ' '}).join('')})
  110. console.log('\n'+args.map(function(a,i){return a+white.substr(a.length)+argsRules.description[keys[i]]+((keys[i] in argsRules.default)?(' [DEFAULT:'+argsRules.default[keys[i]]+']'):'')}).join('\n'));
  111. process.exit(0);
  112. }
  113.  
  114. if (args.init)
  115. args.forceupdate=true;
  116.  
  117. if (args.legacyconvert){
  118. args.checkindex=false,
  119. args.forceupdate=true;
  120. }
  121.  
  122. errors=[];
  123.  
  124. global.missing={
  125. file:[],
  126. index:[]
  127. }
  128.  
  129. global.diff=[]
  130.  
  131. indexes=(args.all?
  132. (!fs.existsSync(args.mountpoint)?(function(){throw 'Could not find default drives location'})():
  133. [].concat.apply([],fs.readdirSync(args.mountpoint).map(function(p){
  134. return args.mountpoint+'/'+p+'/'+args.indexdir;
  135. }).filter(function(p){
  136. return fs.existsSync(p)&&fs.statSync(p).isDirectory();
  137. }).map(function(p){
  138. return fs.readdirSync(p).filter(function(f){
  139. return f.match(/.*\.extindex$/)
  140. }).map(function(f){
  141. console.log('Found: ',p+'/'+f);
  142. return p+'/'+f
  143. })
  144. }))):
  145. args._).filter(function(p,i){
  146. if (args.init^(!fs.existsSync(p))){
  147. console.error('Index '+(args.init?'already exists':'does not exist')+' - aborting',p);
  148. process.exit(10);
  149. }
  150. if (args.init&&!fs.existsSync(args['--'][i])){
  151. console.error('Could not find base dir / source md5index for '+p)
  152. process.exit(13);
  153. }
  154. return true;
  155. }).map(function(p,i){
  156. if (!args.init){
  157. args.verbose&&console.log('Reading index... (',p,')');
  158. var bfile = fs.readFileSync(fs.realpathSync(p)),
  159. sha512 = bfile.slice(0,128),
  160. content = bfile.slice(128);
  161.  
  162. var sha = child_process.spawnSync('sha512sum',[],{input:content})
  163. if (sha.error||sha.status){
  164. console.error(sha.error||'sha512sum exit status code: '+sha.status+' '+String(sha.stderr));
  165. process.exit(45);
  166. }
  167.  
  168. if (String(sha.stdout).substr(0,128)!=sha512){
  169. if (args.checkindex){
  170. console.error('Index sha512 check failed - index is probably damaged - aborting\n',sha512,'\n',String(sha.stdout).substr(0,128))
  171. process.exit(14);
  172. } else
  173. console.warn('Index sha512sum check failed - index is probably damaged - forced ignore\n',sha512,'\n',String(sha.stdout).substr(0,128));
  174. }
  175. try{
  176. var pind = JSON.parse(args.legacyconvert?bfile:content),
  177. ostr = String(args.legacyconvert?bfile:content),
  178. str = JSON.stringify(pind,null,2);
  179.  
  180. for (var i = 0, l = str.length ; i < l ; i++)
  181. if (str[i]!=ostr[i]){
  182. console.error('Soft bug occured - it\'s serious bug and probably classifies as node bug or linux memcache bug. Should be reported');
  183. throw ('Original string and reparsed don\'t match at '+i+' byte - system string conversion malfunction - abtorting')
  184. }
  185.  
  186. if (!(typeof pind == 'object' && !Array.isArray(pind) && 'index' in pind))
  187. throw 'Invalid index file';
  188.  
  189. var curpath = fs.realpathSync(p);
  190. if (pind.extindex!=curpath)
  191. console.warn('Index has been moved from its previous location - it may result in errors');
  192. pind.extindex=curpath;
  193.  
  194. if (!pind.index){
  195. pind.index=[];
  196. for (var i = 0, l = pind.subindexnum ; i < l ; ++i)
  197. pind.index.push(pind.extindex+'.part'+i);
  198.  
  199. pind.index=[].concat.apply([],pind.index.map(function(p,i){
  200. args.verbose&&console.log('Reading subindex... (',p,')');
  201. if (!fs.existsSync(p))
  202. throw 'Expected subindex does not exist';
  203.  
  204. var bfile = fs.readFileSync(p),
  205. sha512 = bfile.slice(0,128),
  206. content = bfile.slice(128);
  207.  
  208. var sha = child_process.spawnSync('sha512sum',[],{input:content});
  209. if(sha.error||sha.status)
  210. throw sha.error||'sha512sum exit status code for subindex '+i+': '+sha.status+' '+String(sha.stderr);
  211. if((sha=String(sha.stdout).substr(0,128))!=sha512){
  212. if (args.checkindex)
  213. throw 'Subindex sha512 check failed - subindex is probably damaged - aborting\n'+sha512+'\n'+sha;
  214. else
  215. console.warn('Subindex sha512 check failed - subindex is probably damaged - forced ignore\n'+sha512+'\n'+sha);
  216. }
  217.  
  218. var sind = JSON.parse(content),
  219. ostr=String(content),
  220. str = JSON.stringify(sind,null,2);
  221.  
  222. for (var i = 0, l = str.length; i < l ; i++)
  223. if (str[i]!=ostr[i]){
  224. console.error('Soft bug occured - it\'s serious bug and probably classifies as node bug or linux memcache bug. Should be reported');
  225. throw ('Original string and reparsed don\'t match at '+i+' byte - system string conversion malfunction - abtorting')
  226. }
  227.  
  228. return sind;
  229.  
  230. }));
  231. }
  232.  
  233. return pind;
  234. } catch (e) {
  235. console.error('Could not read index - aborting',p,e);
  236. process.exit(11);
  237. }
  238. } else {
  239. args.verbose&&console.log('Generating index... (',p,')');
  240. p.match(/\.extindex$/)||((p+='.extindex')&&console.warn('files index should have .extindex extension - renaming'))
  241. if (fs.existsSync(p)){
  242. console.error('Index already exists - aborting',p);
  243. process.exit(12)
  244. }
  245.  
  246. var dirbase=fs.statSync(args['--'][i]).isDirectory()
  247.  
  248. fs.writeFileSync(p,'');
  249. var newIndex = {
  250. extindex:fs.realpathSync(p),
  251. md5index:dirbase?null:fs.realpathSync(args['--'][i]),
  252. base:dirbase?fs.realpathSync(args['--'][i]):null,
  253. index:dirbase?[]:String(fs.readFileSync(args['--'][i])).split('\n').filter(function(line,i){
  254. if (!line.match(/^[0-9a-f]{32} .+/)){
  255. console.warn('Malformed line: ('+i+') "'+line+'"')
  256. return false;
  257. }
  258. return true;
  259. }).map(function(line){
  260. return {
  261. md5:line.substr(0,32),
  262. path:line.substr(34),
  263. stat:null
  264. }
  265. }).filter(function(ent){
  266. if (!fs.existsSync(ent.path)){
  267. console.warn('File '+ent.path+' does not exist - skipped');
  268. return false;
  269. }
  270. return true;
  271. }).map(function(ent){
  272. ent.path=fs.realpathSync(ent.path);
  273. return ent;
  274. }).sort(function(a,b){
  275. return (a.path>b.path?1:(a.path<b.path?-1:0))
  276. }).map(function(ent,i,arr){
  277. args.verbose&&arr.length>100000&&(i%10000==0)&&console.log('[stat:] ',(i*100/arr.length).toPrecision(3)+'%');
  278.  
  279. ent.stat=JSON.parse(JSON.stringify(fs.statSync(ent.path)))
  280. ent.path=ent.path.split('/').filter(function(s){return s!=''})
  281.  
  282. return ent
  283. })
  284. };
  285.  
  286. if (!newIndex.base){
  287. var ind=newIndex.index,
  288. longTree=ind[0].path.slice(0,-1);
  289. for (var i = 0, l = ind.length ; i < l ; i++){
  290. for (var j = 0, ll = ind[i].path.length-1 ; j < ll ; j++)
  291. if (longTree[j]!=ind[i].path[j])
  292. break;
  293. longTree=longTree.slice(0,j);
  294. }
  295. newIndex.base='/'+longTree.join('/');
  296. newIndex.index=ind.map(function(ent){
  297. return {
  298. stat:ent.stat,
  299. md5:ent.md5,
  300. path:ent.path.slice(longTree.length)
  301. }
  302. })
  303. }
  304. return newIndex;
  305. }
  306. }).filter(function validate(index){
  307. args.verbose&&console.log('Refreshing... (',index.extindex,')');
  308.  
  309. //fetch current list of files
  310.  
  311. var realindex = child_process.spawnSync('find',[index.base,'-type','f'],{cwd:'/'});
  312. if (realindex.error||realindex.status){
  313. console.error(realindex.error||('find exit status code: '+realindex.status))
  314. process.exit(40);
  315. }
  316. var strrealindex=String(realindex.stdout);
  317. if (strrealindex!=realindex.stdout){
  318. console.error('Soft bug occured - it\'s serious bug and probably classifies as node bug or linux memcache bug. Should be reported');
  319. throw ('Original string and reparsed don\'t match at '+i+' byte - system string conversion malfunction - abtorting')
  320. }
  321. realindex=strrealindex.split('\n').sort(function(a,b){
  322. return (a>b?1:(a<b?-1:0))
  323. }).filter(function(p){
  324. if (!fs.existsSync(p)){
  325. if (p!=''){
  326. console.error('Something went wrong - find returned non-existing file - aborting ',p)
  327. process.exit(41);
  328. }
  329. return false;
  330. }
  331. return true
  332. }).map(function(p){
  333. return p.split('/').filter(function(s){return s!=''})
  334. });
  335.  
  336.  
  337. var missingFile=[],
  338. missingIndex=[];
  339.  
  340. var diff=[];
  341.  
  342. //find missing files
  343.  
  344. index.index=index.index.filter(function(ent){
  345. if (!fs.existsSync(index.base+'/'+ent.path.join('/')))
  346. return (missingFile.push(ent)&&false);
  347. return true;
  348. })
  349.  
  350. //find missing index entries
  351.  
  352. for (i=0,j=0,l=index.index.length,ll=realindex.length; i<l&&j<ll ;){
  353. var pi = index.base+'/'+index.index[i].path.join('/'),
  354. pr = '/'+realindex[j].join('/');
  355.  
  356. if (pi==pr){
  357. i++;
  358. j++;
  359. } else if (pi>pr){
  360. missingIndex.push(pr)
  361. j++;
  362. } else { //(pi<pr)
  363. console.error('something went wrong - there should be no missing files at this step (possible BUG) - aborting');
  364. process.exit(60);
  365. }
  366. }
  367.  
  368. if (j!=ll)
  369. for (;j<ll;j++)
  370. missingIndex.push('/'+realindex[j].join('/'));
  371.  
  372. if (i!=l)
  373. for (;i<l;i++){
  374. console.error('something went wrong - there should be no missing files at this step (possible BUG) - aborting');
  375. process.exit(61);
  376. }
  377.  
  378.  
  379. //compare stat
  380.  
  381. var md5queue=[];
  382. args.verbose&&console.log('Comparing stat...');
  383. for (var i = 0, l = index.index.length; i < l ; i++){
  384.  
  385. args.verbose&&l>100000&&(i%10000==0)&&console.log('[stat:] ',(i*100/l).toPrecision(3)+'%');
  386. var ent=index.index[i],
  387. p=index.base+'/'+ent.path.join('/'),
  388.  
  389. newStat=JSON.parse(JSON.stringify(fs.statSync(p))),
  390. oldStat=ent.stat;
  391.  
  392. var d={};
  393.  
  394. var cmpKeys=['mode','uid','gid','size','mtime'];
  395.  
  396. for (var j=0,ll=cmpKeys.length;j<ll;j++)
  397. if (oldStat[cmpKeys[j]]!=newStat[cmpKeys[j]])
  398. d[cmpKeys[j]]=[oldStat[cmpKeys[j]],newStat[cmpKeys[j]]];
  399.  
  400. if (Object.keys(d).length)
  401. diff.push({ent:ent,diff:d})
  402.  
  403. var junkDiff=0;
  404. if (args.careless){
  405. ('gid' in d)&&junkDiff++;
  406. ('uid' in d)&&junkDiff++;
  407. ('mode' in d)&&junkDiff++;
  408. }
  409.  
  410. if (args.deep||
  411. ((Object.keys(d).length>junkDiff||
  412. ent.md5=='########### skipped ############'||
  413. ent.md5=='############ ERROR #############')&&
  414. !args.fast))
  415. md5queue.push({ent:ent,diff:d});
  416. }
  417.  
  418. if (args.importcorrupted){
  419. if (!fs.existsSync(args.importcorrupted))
  420. throw 'Could not find corrupted files list';
  421.  
  422. var list = String(fs.readFileSync(args.importcorrupted)).split('\n').filter(function(p){
  423. return !args.md5sumcorrupted||p.match(/: FAILED$/);
  424. }).filter(function(p){
  425. return p.indexOf(index.base)==0
  426. }).map(function(p){
  427. return (args.md5sumcorrupted?p.replace(/: FAILED$/,''):p).split('/').filter(function(s){return s!=''}).slice(index.base.split('/').filter(function(s){return s!=''}).length)
  428. })
  429.  
  430. var files = [].concat.apply([],list.map(function(p){
  431. var valid=index.index;
  432. for (var i = 0, l = p.length ; i < l ; i++)
  433. valid=valid.filter(function(ent){
  434. return ent.path[i]==p[i];
  435. })
  436. return valid;
  437. })).filter(function(e,i,arr){
  438. return i==arr.indexOf(e)
  439. });
  440.  
  441. args.verbose&&console.log('Following files will be force reloaded in this index: ( '+index.base+' )\n'+files.map(function(ent){return ' '+ent.path.join('/')}).join('\n'));
  442.  
  443. files.forEach(function(ent){
  444. if (md5queue.filter(function(cqueue){return cqueue.ent==ent}).length==0)
  445. md5queue.push({ent:ent,diff:{}});
  446. })
  447. }
  448.  
  449. for (var i = 0, l = md5queue.length, md5queueSize=0 ; i < l ; i++)
  450. md5queueSize+=(md5queue[i].diff.size?md5queue[i].diff.size[1]:md5queue[i].ent.stat.size)
  451.  
  452. for (var i = 0, l = md5queue.length, md5queueProgress=0 ; i < l ; i++){
  453. var cqueue=md5queue[i];
  454.  
  455. args.verbose&&
  456. (l<500||i%50==0||(cqueue.diff.size?cqueue.diff.size[1]:cqueue.ent.stat.size)>50*1024*1024)&&
  457. console.log('[md5:] ',i+1+'/'+l,(md5queueProgress*100/md5queueSize).toPrecision(3)+'% '+(l<500?cqueue.ent.path.join('/'):''))
  458.  
  459. var ret=child_process.spawnSync('md5sum',[index.base+'/'+cqueue.ent.path.join('/')],{cwd:'/'});
  460. if (ret.error||ret.status){
  461. console.error(ret.error||'md5sum exit status code: '+ret.status+' '+String(ret.stderr));
  462. process.exit(43);
  463. }
  464. ret=String(ret.stdout).replace('\n','').substr(0,32);
  465.  
  466. if (ret!=cqueue.ent.md5){
  467. if (Object.keys(cqueue.diff).length==0)
  468. diff.push(cqueue);
  469. cqueue.diff.md5=[cqueue.ent.md5,ret];
  470. }
  471.  
  472. md5queueProgress+=(cqueue.diff.size?cqueue.diff.size[1]:cqueue.ent.stat.size);
  473. }
  474.  
  475.  
  476. //apply diff
  477.  
  478. //process.exit(0);
  479.  
  480. diff.forEach(function(d){
  481. var keys=Object.keys(d.diff).filter(function(k){return k!='md5'});
  482. for (var i = 0, l = keys.length ; i < l ; i++)
  483. d.ent.stat[keys[i]]=d.diff[keys[i]][1];
  484. if (d.diff.md5)
  485. d.ent.md5=d.diff.md5[1];
  486. });
  487.  
  488.  
  489. //add new files to index
  490.  
  491. if (missingIndex.length){
  492.  
  493. args.verbose&&console.log('Updating extindex... ('+missingIndex.length+' new files)')
  494.  
  495. var baselen=index.base.split('/').filter(function(b){return b!=''}).length
  496.  
  497. for (var i = 0, l = missingIndex.length, missingSize=0 ; i < l ; i++)
  498. missingSize+=fs.statSync(missingIndex[i]).size;
  499.  
  500. var missingProgress=0,l=missingIndex.length;
  501.  
  502. missingIndex=missingIndex.map(function(p,i){
  503. var s=fs.statSync(p);
  504.  
  505. args.verbose&&
  506. (l<500||i%50==0||s.size>50*1024*1024)&&
  507. console.log(args.fast?'[stat:] ':'[md5:] ',i+1+'/'+l,(missingProgress*100/missingSize).toPrecision(3)+'% '+(l<500?p:''));
  508.  
  509. var ret='########### skipped ############ '+p
  510. if (!args.fast){
  511. ret=child_process.spawnSync('md5sum',[p],{cwd:'/'});
  512. if (ret.error||ret.status){
  513. console.error(ret.error||'md5sum exit status code: '+ret.status+' '+String(ret.stderr));
  514. //process.exit(44);
  515. ret='############ ERROR ############# '+p
  516. errors.push([ret.status,String(ret.stderr)]);
  517. } else
  518. ret=String(ret.stdout).replace('\n','');
  519. }
  520.  
  521. missingProgress+=s.size;
  522.  
  523. return {
  524. stat:s,
  525. md5:ret.substr(0,32),
  526. path:ret.substr(34).split('/').filter(function(s){return s!=''}).slice(baselen)
  527. }
  528. })
  529.  
  530. index.index=index.index.concat(missingIndex).sort(function(a,b){
  531. var pa=a.path.join('/'),
  532. pb=b.path.join('/');
  533. return (pa>pb?1:(pa<pb?-1:0))
  534. });
  535. }
  536.  
  537. missing.file=missing.file.concat(missingFile);
  538. missing.index=missing.index.concat(missingIndex);
  539. global.diff=global.diff.concat(diff);
  540.  
  541. var now=Date.now(),
  542. basepath=index.extindex.replace(/\/[^\/]*$/,''),
  543. filename=index.extindex.replace(/.*\//,''),
  544. newbase=basepath+'/extindex.old/',
  545. newfile=newbase+filename;
  546.  
  547.  
  548.  
  549. if ((args.forceupdate||!args.fast)&&!args.simulate){
  550. if (!fs.existsSync(newbase))
  551. fs.mkdirSync(newbase);
  552.  
  553. if (missingFile.length){
  554. console.log('Creating missing list... ('+missingFile.length+' files missing)')
  555. fs.writeFileSync(newfile+'.'+now+'.missing',JSON.stringify(missingFile,null,2));
  556. }
  557.  
  558. if (missingIndex.length){
  559. console.log('Creating new files list... ('+missingIndex.length+' new files)')
  560. fs.writeFileSync(newfile+'.'+now+'.new',JSON.stringify(missingIndex,null,2));
  561. }
  562.  
  563. if (diff.length){
  564. console.log('Creating diff list... ('+diff.length+' files different)')
  565. fs.writeFileSync(newfile+'.'+now+'.diff',JSON.stringify(diff,null,2));
  566. }
  567.  
  568. if (args.forceupdate||diff.length||missingIndex.length||missingFile.length){
  569. args.verbose&&console.log('[extindex:] ',index.extindex);
  570.  
  571. var old = fs.readdirSync(newbase)
  572. .filter(function(p){return p.indexOf(filename)==0})
  573. .filter(function(p){return p.match(/\.old\.[0-9]+$/)});
  574. old.filter(function(p){return p.match(/[0-9]+$/)[0]>=args.backup}).forEach(function(p){fs.unlinkSync(newbase+p)});
  575.  
  576. if (args.backup){
  577. for (var b = args.backup-1 ; b > 0 ; b--)
  578. old.filter(function(p){return p.match(/[0-9]+$/)[0]==b}).forEach(function(p){fs.renameSync(newbase+p,newbase+p.replace(/[0-9]+$/,b+1))})
  579. fs.renameSync(index.extindex,newfile+'.old.1');
  580. for (var i = 0, l = index.subindexnum ; i < l ; i++)
  581. fs.renameSync(index.extindex+'.part'+i,newfile+'.part'+i+'.old.1');
  582. }
  583.  
  584. subindexes=[];
  585. while (index.index.length>25000)
  586. subindexes.push(index.index.splice(0,25000));
  587. subindexes.push(index.index);
  588. if(subindexes.length>1){
  589. index.index=null;
  590. index.subindexnum=subindexes.length;
  591. } else {
  592. subindexes=[];
  593. index.subindexnum=0;
  594. }
  595. var filenames=[index.extindex],
  596. filecontents=[index];
  597.  
  598. for (var i = 0, l = subindexes.length ; i < l ; i++){
  599. filenames.push(index.extindex+'.part'+i);
  600. filecontents.push(subindexes[i]);
  601. }
  602.  
  603. for (var i = 0, l = filenames.length ; i < l ; i++){
  604. args.verbose&&console.log('[extindex:] part '+(i+1)+'/'+l);
  605.  
  606. var content = JSON.stringify(filecontents[i],null,2),
  607. sha512;
  608.  
  609. var sha = child_process.spawnSync('sha512sum',[],{input:content})
  610. if (sha.error||sha.status){
  611. console.error(sha.error||'sha512sum exit status code: '+sha.status+' '+String(sha.stderr));
  612. process.exit(45);
  613. }
  614. sha512 = String(sha.stdout).substr(0,128);
  615.  
  616. try {
  617. fs.writeFileSync(filenames[i],sha512+content);
  618. var file = fs.readFileSync(filenames[i]);
  619. if (file!=(sha512+content))
  620. throw ':c';
  621. } catch (e) {
  622. console.error('Error occured during index write - retry')
  623. fs.writeFileSync(filenames[i],sha512+content);
  624. var file = fs.readFileSync(filenames[i]);
  625. if (file!=(sha512+content))
  626. throw 'Could not properly write index file - operation totally failed, all your data is lost and ur doomed :< - aborting';
  627. }
  628. }
  629. if (subindexes.length)
  630. index.index=[].concat.apply([],subindexes)
  631.  
  632. }
  633.  
  634. if (args.export){
  635. var md5indexpath=basepath+'/md5index/'+filename.replace(/\.extindex$/,'.md5index');
  636. args.verbose&&console.log('[md5index:] ',md5indexpath)
  637.  
  638. if (!fs.existsSync(basepath+'/md5index'))
  639. fs.mkdirSync(basepath+'/md5index');
  640.  
  641. if (fs.existsSync(md5indexpath+'.old'))
  642. fs.unlinkSync(md5indexpath+'.old');
  643. if (fs.existsSync(md5indexpath))
  644. fs.renameSync(md5indexpath,md5indexpath+'.old');
  645.  
  646. fs.writeFileSync(md5indexpath,index.index.map(function(ent){
  647. return ent.md5+' '+index.base+'/'+ent.path.join('/');
  648. }).join('\n'))
  649. }
  650. } else
  651. args.verbose&&console.log((args.fast?'Fast':'Simulate')+' mode - log write skipped');
  652.  
  653. return true;
  654. });
  655.  
  656.  
  657. Object.defineProperties(global,{
  658. shortdiff:{
  659. get:function(){
  660. return diff.map(function(d){
  661. return Object.keys(d.diff).sort().join(',') + ' -- ' + d.ent.path.join('/');
  662. })
  663. }
  664. },
  665.  
  666. shortmissing:{
  667. get:function(){
  668. return missing.file.map(function(ent){
  669. return ent.path.join('/');
  670. })
  671. }
  672. },
  673.  
  674. shortnew:{
  675. get:function(){
  676. return missing.index.map(function(ent){
  677. return ent.path.join('/');
  678. })
  679. }
  680. }
  681. });
  682.  
  683.  
  684. if (args.interactive||(args.halfinteractive&&missing.file.length+missing.index.length+diff.length>0))
  685. require('/usr/lib/node_modules/interactive/').start(this);
  686. else
  687. args.verbose&&console.log('Nothing to do - exiting');
  688.  
  689.  
  690. function buildExtendedTree(index){
  691. var extIndex={
  692. md5index:index.md5index,
  693. basePath:index.base,
  694. tree:null
  695. },
  696. tree={};
  697.  
  698. (function buildDirTree(ind,tree){
  699.  
  700. for (var i = 0, l = ind.length ; i<l ; i++){
  701.  
  702. }
  703.  
  704. })(index.index,tree);
  705.  
  706. extIndex.tree=tree;
  707. return extIndex;
  708. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!