SHARE
TWEET

put-assets

a guest Feb 4th, 2016 116 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. var winston = require('winston');
  2. var commander = require('commander');
  3. var readline = require('readline');
  4. var fs = require('fs');
  5. var RateLimiter = require('limiter').RateLimiter;
  6. var Amplience = require('node-amplience-api');
  7. var EOL = require('os').EOL;
  8.  
  9. commander
  10.     .version('0.0.0')
  11.     .usage('[options] - input & output can be piped or directed to a file using --input and --output')
  12.     .option('-l, --log [value]', 'output file for errors and warnings')
  13.     .option('-i, --input [value]', 'input file if not std-in')
  14.     .option('-o, --output [value]', 'output file if not std-out')
  15.     .option('-c, --concurrent-api-calls [value]', 'how many concurrent api requests to process (defaults to 5)')
  16.     .option('-r, --rate <n>', 'api requests per minute rate')
  17.     .option('-a, --api [value]', 'api url e.g. http://qa-3-dam-api-ssl.adis.ws')
  18.     .option('-u --username [value]', 'api username')
  19.     .option('-p --password [value]', 'api password')
  20.     .option('-m, --mode [value]', 'rename mode, valid options are renameUnique or overwrite. Default is renameUnique.', /^(renameUnique|overwrite)$/i)
  21.     .parse(process.argv);
  22.  
  23. if (commander.log) {
  24.     //Log status messages to file
  25.     winston.add(winston.transports.File, { filename: commander.log });
  26.     winston.remove(winston.transports.Console);
  27. }
  28.  
  29. var concurrentApiCalls = commander['concurrent-api-calls'] || 5;
  30. // 60 requests per minute
  31. var rate = commander.rate || 60;
  32. var limiter = new RateLimiter(rate, 'minute');
  33.  
  34. winston.info('starting PUT /assets requests at a rate of ' + rate + ' requests per minute');
  35.  
  36. var API_USERNAME = process.env.AMP_USERNAME || commander.username;
  37. var API_PASSWORD = process.env.AMP_PASSWORD || commander.password;
  38. var apiUrl = commander.api || process.env.AMP_API || 'https://qa-3-dam-api-ssl.adis.ws';
  39.  
  40. var api = new Amplience({ retries: 0, base: apiUrl, logLevel: 'none' });
  41. var out = commander.output ? fs.createWriteStream(commander.output) : process.stdout;
  42. var queue = [];
  43. var complete = false;
  44. var isDigestingLocked = false;
  45.  
  46. function lockDigest() {
  47.     isDigestingLocked = true;
  48. }
  49.  
  50. function unlockDigest() {
  51.     isDigestingLocked = false;
  52.     digestQueue();
  53. }
  54.  
  55. function digestQueue() {
  56.     if (isDigestingLocked) { return; }
  57.  
  58.     //Every line scanned and digest complete
  59.     if (complete && queue.length == 0) {
  60.         winston.info('complete');
  61.         return;
  62.     }
  63.  
  64.     if (queue.length >= concurrentApiCalls || complete) {
  65.         //Submit
  66.         var batch = queue.splice(0, concurrentApiCalls);
  67.         submit(batch);
  68.     }
  69. }
  70.  
  71. function output(line) {
  72.     out.write(line);
  73. }
  74.  
  75. function submit(batch) {
  76.     //Stop any more activity until I am done
  77.     lockDigest();
  78.  
  79.     var remaining = [].concat(batch);
  80.     remaining.forEach(function (x) {
  81.         //Null means failed. Otherwise the id of the inserted item will be set
  82.         x.insertId = null;
  83.     });
  84.  
  85.     //all puts completed or failed
  86.     function checkPutComplete() {
  87.         if (remaining.length == 0) {
  88.             //Output ids
  89.             for (var i = 0; i < batch.length; i++) {
  90.                 var batchItem = batch[i];
  91.                 if (batchItem.insertId == null) {
  92.                     //This asset put failed
  93.                     winston.error('put failed', batchItem);
  94.                     output(EOL);
  95.                 } else {
  96.                     output(batchItem.insertId + EOL);
  97.                 }
  98.             }
  99.             unlockDigest();
  100.         }
  101.     }
  102.  
  103.     function removeItemFromRemaining(items) {
  104.         remaining.splice(remaining.indexOf(items), 1);
  105.         checkPutComplete();
  106.     }
  107.  
  108.     function putItem(items) {
  109.         //Items is an array, if the first one fails move onto the next one
  110.         var index = 0;
  111.         function tryNext() {
  112.             var item = items[index];
  113.             index++;
  114.             if (item == null) {
  115.                 //Failed, remove me
  116.                 removeItemFromRemaining(items);
  117.                 return;
  118.             }
  119.  
  120.             var request = {
  121.                 mode: commander.mode || 'renameUnique',
  122.                 assets: [item]
  123.             };
  124.  
  125.             limiter.removeTokens(1, function (err, remainingRequests) {
  126.                 api.put('/assets', request).then(function (response) {
  127.                     if (response.error) {
  128.                         tryNext();
  129.                     } else {
  130.                         var result = response.obj;
  131.                         //Overall request status
  132.                         if (result.status != 'success' ||
  133.                             result.content == null ||
  134.                             result.content[0].status != 'succeeded') {
  135.                             tryNext();
  136.                         } else {
  137.                             //Done!
  138.                             items.insertId = result.content[0].id;
  139.                             removeItemFromRemaining(items);
  140.                         }
  141.                     }
  142.                 }).catch(function (err) {
  143.                     //problem! move onto the next asset
  144.                     tryNext();
  145.                 });
  146.             });
  147.         }
  148.  
  149.         tryNext();
  150.     }
  151.  
  152.     remaining.forEach(function (x) { putItem(x); });
  153.     //invoke asset put
  154.     //unlockDigest();
  155.  
  156. }
  157.  
  158.  
  159. var rl = null;
  160.  
  161. function readInput() {
  162.     //Read std-in line by line
  163.     rl = readline.createInterface({
  164.         input: commander.input ? fs.createReadStream(commander.input) : process.stdin,
  165.         output: commander.output ? fs.createWriteStream(commander.output) : process.stdout,
  166.         terminal: false
  167.     });
  168.  
  169.  
  170.     rl.on('line', function (line) {
  171.         if (line == '') {
  172.             return;
  173.         }
  174.  
  175.         //Line received, is it valid JSON
  176.         try {
  177.             var parsed = JSON.parse(line);
  178.         } catch (e) {
  179.             winston.error('invalid json', line);
  180.             return;
  181.         }
  182.  
  183.         queue.push(parsed);
  184.         digestQueue();
  185.     });
  186.  
  187.     rl.on('close', function () {
  188.         //No more input
  189.         complete = true;
  190.         digestQueue();
  191.     });
  192.  
  193.     rl.resume();
  194. }
  195.  
  196. function bootstrap() {
  197.     api.login(API_USERNAME, API_PASSWORD).then(function () {
  198.         winston.info('acquired authentication token');
  199.         //Start working
  200.         readInput();
  201.     }, function (err) {
  202.         winston.error('unable to authenticate with api');
  203.         process.exit(1);
  204.     });
  205. }
  206.  
  207. bootstrap();
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy. OK, I Understand
 
Top