parser.py

import csv

from models import *


import os


def toFloat(s):
    s = s.replace(',', '.')
    try:
        return float(s)
    except:
        if s == '0,00' or s == '0.00':
            return 0.00
        if s == '':
            return None
        return s

def toInt(s):
    s = s.replace(',', '.')
    try:
        return int(s)
    except:
        if s == '0.00':
            return 0
        if s == '':
            return None
        return s

def toSmallInt(s):
    s = s.replace(',', '.')
    try:
        s = int(s)
        if s > 32767:
            return 32767
        return s
    except:
        if s == '0.00':
            return 0
        if s == '':
            return None
        return s


for dirname, _, filenames in os.walk('files'):
    for filename in filenames:
        data = []
        file = os.path.join(dirname, filename)
        print(file)
        with open(file, "r") as f:
            reader = csv.reader(f, delimiter=";")
            index = 0
            for i, line in enumerate(reader):
                index = index + 1
                if index == 1:
                    continue
                dictAppend = {
                    'id_ref': toInt(line[0]),
                    'region_id': line[1],
                    'area_id': line[2],
                    'city_id': line[3],
                    'street_id': line[4],
                    'houseguid': line[18],
                    'management_organization_id': toInt(line[19]),
                    'built_year': toSmallInt(line[20]),
                    'exploitation_start_year': toSmallInt(line[21]),
                    'project_type': line[22],
                    'house_type': line[23],
                    'is_alarm': line[24],
                    'method_of_forming_overhaul_fund': line[25],
                    'floor_count_max': toSmallInt(line[26]),
                    'floor_count_min': toSmallInt(line[27]),
                    'entrance_count': toSmallInt(line[28]),
                    'elevators_count': toSmallInt(line[29]),
                    'energy_efficiency': line[30],
                    'quarters_count': toSmallInt(line[31]),
                    'living_quarters_count': toSmallInt(line[32]),
                    'unliving_quarters_count': toSmallInt(line[33]),
                    'area_total': toFloat(line[34]),
                    'area_residential': toFloat(line[35]),
                    'area_non_residential': toFloat(line[36]),
                    'area_common_property': toFloat(line[37]),
                    'area_land': toFloat(line[38]),
                    'parking_square': toFloat(line[39]),
                    'playground': toSmallInt(line[40]),
                    'sportsground': toSmallInt(line[41]),
                    'other_beautification': line[42],
                    'foundation_type': line[43],
                    'floor_type': line[44],
                    'wall_material': line[45],
                    'basement_area': toFloat(line[46]),
                    'chute_type': line[47],
                    'chute_count':  toInt(line[48]),
                    'electrical_type': line[49],
                    'electrical_entries_count':  toSmallInt(line[50]),
                    'heating_type': line[51],
                    'hot_water_type': line[52],
                    'cold_water_type': line[53],
                    'sewerage_type': line[54],
                    'sewerage_cesspools_volume': toFloat(line[55]),
                    'gas_type': line[56],
                    'ventilation_type': line[57],
                    'firefighting_type': line[58],
                    'drainage_type': line[59],
                }
                data.append(dictAppend)
        try:
            ReformaOpendata.insert_many(data).execute()
        except BaseException as e:
            print(str(e))
            exit()
        print("OK file=", file)
print("OK ALL")