Advertisement
Guest User

Untitled

a guest
Dec 7th, 2016
80
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 1.28 KB | None | 0 0
  1.  with gzip.open(filepath, 'rt', newline='') as gzip_file:
  2.                 dr = csv.DictReader(gzip_file)  # comma is default delimiter
  3.                 chunksize = oci.settings.CHUNKSIZE
  4.                 total_chunks = math.ceil(rows_count / chunksize)
  5.  
  6.                 # for chunk in tqdm(oci.utils.chunker.gen_chunks(dr, chunksize=oci.settings.CHUNKSIZE),
  7.                 #                   total=total_chunks):
  8.                 operations = []
  9.                 for row in tqdm(dr, total=rows_count):
  10.                     cell = {
  11.                         'mcc': int(row['mcc']),
  12.                         'mnc': int(row['net']),
  13.                         'lac': int(row['area']),
  14.                         'cell': int(row['cell'])
  15.                     }
  16.                     location = {
  17.                         'lat': float(row['lat']),
  18.                         'lon': float(row['lon'])
  19.                     }
  20.                     operations.append(UpdateOne(cell, {'$set': {source.source_name: location}}, upsert=True))
  21.  
  22.                     if len(operations) == chunksize:
  23.                         locations.bulk_write(operations, ordered=False)
  24.                         operations = []
  25.                 if len(operations) > 0:
  26.                     locations.bulk_write(operations, ordered=False)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement