geilt

Slice Data set for Smaller Queries.

Dec 5th, 2014
175
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1.   queue: function(args, callback){
  2.         var db = this.db;
  3.         var length = args.length;
  4.         var slices = [];
  5.         if(length > 100){
  6.             console.log('mailer.model', 'Splitting Up Large Queue Insert');
  7.             var totalLoops = Math.ceil(length / 100);
  8.             var lastSlice = length - ( totalLoops * 100);
  9.             console.log(length, totalLoops, lastSlice);
  10.             var range = _.range(0, totalLoops);
  11.             range.forEach(function(num, index){
  12.                 var start = num === 0 ? num : ((num) * 100) + 1;
  13.                 var end = num === 0 ? 100 : start + 99;
  14.                 if(end < lastSlice) end = start + lastSlice + 1;
  15.                 slices.push(args.slice(start, end));
  16.                 console.log(start, 'to', end);
  17.             });
  18.         } else {
  19.             slices = [args];
  20.         }
  21.  
  22.         slices.forEach(function(slice, index){
  23.             console.log('Slice', 'index');
  24.             var keys = db.mysql.getBulkInsertKeys(slice);
  25.             var data = db.mysql.getBulkInsertData(slice);
  26.             var query = "INSERT INTO mailer_logs ( ?? ) VALUES ?";
  27.  
  28.             db.mysql.query(query,[keys, data], function(err, result) {
  29.                 //console.log(err, result);
  30.                 callback(err, result);
  31.                 if(err) console.log('mailer.model', 'Queue Insertion Problem', err, response);
  32.             });
  33.         });
  34.     },
Advertisement
Add Comment
Please, Sign In to add comment