Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- with gzip.open(filepath, 'rt', newline='') as gzip_file:
- dr = csv.DictReader(gzip_file) # comma is default delimiter
- chunksize = oci.settings.CHUNKSIZE
- total_chunks = math.ceil(rows_count / chunksize)
- # for chunk in tqdm(oci.utils.chunker.gen_chunks(dr, chunksize=oci.settings.CHUNKSIZE),
- # total=total_chunks):
- operations = []
- for row in tqdm(dr, total=rows_count):
- cell = {
- 'mcc': int(row['mcc']),
- 'mnc': int(row['net']),
- 'lac': int(row['area']),
- 'cell': int(row['cell'])
- }
- location = {
- 'lat': float(row['lat']),
- 'lon': float(row['lon'])
- }
- operations.append(UpdateOne(cell, {'$set': {source.source_name: location}}, upsert=True))
- if len(operations) == chunksize:
- locations.bulk_write(operations, ordered=False)
- operations = []
- if len(operations) > 0:
- locations.bulk_write(operations, ordered=False)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement