Advertisement
Guest User

Untitled

a guest
Jan 25th, 2012
126
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 3.24 KB | None | 0 0
  1. def prepare_cql(cql, params):
  2.    
  3.     """
  4.     Use the cql cursor prepare method to replace the params before we add it to the batch
  5.     """
  6.     from cql import cursor 
  7.     return cursor.prepare(cql, params)
  8.  
  9. def main(dry_run=False, batch_size=10, rows=10000, cols=1, consistency='ONE'):
  10.    
  11.     import random
  12.     import datetime
  13.     import cql
  14.     import time
  15.     import os
  16.     import string
  17.    
  18.     statements = ''
  19.    
  20.     keyspace = 'test_case'
  21.     cf = 'testing'
  22.    
  23.     conn = cql.connect(host='localhost', port=int('9160'), keyspace=keyspace)
  24.     cursor = conn.cursor()
  25.    
  26.     if not dry_run:
  27.         cursor.execute('CREATE KEYSPACE :ks WITH strategy_class = :strat AND strategy_options:replication_factor = 1',{'ks':keyspace, 'strat':'SimpleStrategy'})
  28.         cursor.execute('CREATE COLUMNFAMILY :cf (KEY text PRIMARY KEY)',{'cf':cf})
  29.    
  30.     start = datetime.datetime.now()
  31.     print 'Started: '+str(start)
  32.    
  33.     for i in range(1, rows+1):
  34.        
  35.         key = 'testAutoInsert'+str(i)
  36.        
  37.         #build a dictionary of the test data
  38.         test_data = {u'testColumn_'+str(i):''.join(random.choice(string.ascii_uppercase) for x in range(64)) for i in range(cols)}
  39.        
  40.         #get the keys as a list    
  41.         data_fields = test_data.keys()
  42.         #get the values as a matching list
  43.         data_values = test_data.values()
  44.        
  45.         cql = 'INSERT INTO :column_family (KEY, :c' + ', :c'.join([str(c) for c in range(len(data_fields))]) +') VALUES (:key, :v' + ', :v'.join([str(v) for v in range(len(data_values))]) +') USING TIMESTAMP :ts '
  46.        
  47.         params = {'ts':int(time.time() * 1e9), 'key':key, 'column_family':cf}
  48.         #update the parameters with the dictionary key names
  49.         params.update({'c' + str(cnt):data_fields[cnt] for cnt in range(len(data_fields))})
  50.         #update the parameters with the dictionary values
  51.         params.update({'v' + str(cnt):data_values[cnt] for cnt in range(len(data_values))})
  52.        
  53.         trimmed_cql = prepare_cql(cql, params).strip()
  54.         final_cql = trimmed_cql if trimmed_cql[-1:] == ';' else trimmed_cql + ';'
  55.        
  56.         statements = statements + final_cql
  57.        
  58.         if i % batch_size == 0:
  59.             exec_cql = 'BEGIN BATCH USING CONSISTENCY ' + consistency + ' ' + statements + ' APPLY BATCH'
  60.             if not dry_run:
  61.                 cursor.execute(exec_cql)
  62.             statements = ''
  63.             print 'Inserted Batch Number '+str(rows - (rows - i))
  64.    
  65.    
  66.     end = datetime.datetime.now()
  67.    
  68.     print 'Ended: '+str(end)
  69.    
  70.     print 'Batch Size: '+str(batch_size)
  71.     print 'Rows Inserted: '+str(rows)
  72.     print 'Columns Inserted per Row: '+str(cols)
  73.     print 'Time Took:'+str(end-start)
  74.    
  75.  
  76. if __name__ == '__main__':
  77.    
  78.     import argparse
  79.    
  80.     parser = argparse.ArgumentParser(description='Script used to bench batch inserts for CQL')
  81.     parser.add_argument('-d', '--dry_run', action='store_true', help='In this mode we do not connect to the DB or insert anything in the DB. Default: False')
  82.     parser.add_argument('-b', '--batch_size', type=int, default=10,  help='Size of the batch insert we do. DEFAULT: 100')
  83.     parser.add_argument('-r', '--rows', type=int, default=10000,  help='How many rows to insert. DEFAULT: 10000')
  84.     parser.add_argument('-c', '--cols', type=int, default=1,  help='How many columns per row to insert. DEFAULT: 1')
  85.     parser.add_argument('-cs', '--consistency', type=str, default='ONE',  help='Consistency level to use for the batch. DEFAULT: ONE')
  86.     args = parser.parse_args()
  87.    
  88.     main(**args.__dict__)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement