Guest User

Untitled

a guest
Mar 25th, 2019
67
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 4.58 KB | None | 0 0
  1. import configparser
  2. import os
  3. import csv
  4. import dateutil.parser
  5. import sys
  6. import psycopg2
  7. import time
  8.  
  9. class TemporalRisk:
  10. def __init__(self):
  11. self.config = configparser.ConfigParser()
  12. self.config.read('config.cfg')
  13.  
  14. self.directory = self.config.get('FILE CONFIG', 'Directory')
  15. self.file_delimiter = self.config.get('FILE CONFIG', 'Delimiter')
  16.  
  17. def read_data(self):
  18. input_file_name = self.config.get('FILE CONFIG', 'input_file')
  19. files = os.listdir(self.directory)
  20.  
  21. if input_file_name in files:
  22. input_file = open(os.path.join(self.directory, input_file_name), 'rt', encoding="utf8")
  23. input_file_reader = csv.reader(input_file, delimiter=self.file_delimiter)
  24. data = []
  25.  
  26. for row in input_file_reader:
  27. point = {'account_id': int(row[0]), 'trip_id': int(row[1]), 'lat': float(row[2]), 'lon': float(row[3])}
  28. data.append(point)
  29.  
  30. return data
  31. else:
  32. print("Specified files is not present in the given location")
  33. sys.exit(1)
  34.  
  35. def get_connection(self):
  36. try:
  37. conn = psycopg2.connect("dbname={0} user={1} host={2} password={3} port={4}".format(
  38. self.config.get('DB CONFIG', 'DBName'),
  39. self.config.get('DB CONFIG', 'User'),
  40. self.config.get('DB CONFIG', 'Host'),
  41. self.config.get('DB CONFIG', 'Password'),
  42. self.config.get('DB CONFIG', 'Port')
  43. ))
  44. return conn
  45. except psycopg2.OperationalError as e:
  46. print("Error in establishing connection\n%s" % e)
  47. sys.exit(1)
  48.  
  49. def process_data(self, conn, data):
  50. data_tup = tuple(data)
  51. cursor = conn.cursor()
  52.  
  53. # Create temporary table
  54. cursor.execute("select tn_create_temp_trip_table()")
  55.  
  56. # Insert data to temporary table
  57. cursor.executemany("insert into trip_locations values(%(account_id)s, %(trip_id)s, %(lat)s, %(lon)s)", data_tup)
  58.  
  59. # Build query to score risk
  60. query = """SELECT * from
  61. public.make_linestring();"""
  62.  
  63. cursor.execute(query, (
  64. self.config.get('APP CONFIG', 'tolerance_distance_m'),
  65. self.config.get('APP CONFIG', 'tolerance_duration_s'),
  66. self.config.get('APP CONFIG', 'tolerance_speed_m_s'),
  67. ))
  68.  
  69.  
  70.  
  71. query1 = """SELECT * from public.input_waypoints;"""
  72.  
  73. cursor.execute(query1, (
  74. self.config.get('APP CONFIG', 'tolerance_distance_m'),
  75. self.config.get('APP CONFIG', 'tolerance_duration_s'),
  76. self.config.get('APP CONFIG', 'tolerance_speed_m_s'),
  77. ))
  78.  
  79. # Getting the results
  80. result = cursor.fetchall()
  81.  
  82. conn.close()
  83.  
  84. return result
  85.  
  86. def write_data(self, result):
  87. conn = psycopg2.connect("dbname={0} user={1} host={2} password={3} port={4}".format(
  88. self.config.get('DB CONFIG', 'DBName'),
  89. self.config.get('DB CONFIG', 'User'),
  90. self.config.get('DB CONFIG', 'Host'),
  91. self.config.get('DB CONFIG', 'Password'),
  92. self.config.get('DB CONFIG', 'Port')
  93. ))
  94. outfile = open(os.path.join(self.directory, self.config.get('FILE CONFIG', 'output_file')), 'w',
  95. encoding="utf8", newline='')
  96. outfile_writer = csv.writer(outfile, delimiter=self.file_delimiter)
  97. outfile_writer.writerow(["Account id","route_risk_score", "start_time", "end_time", "time_taken"])
  98. for row in result:
  99. cursor = conn.cursor()
  100.  
  101. # Create temporary table
  102. cursor.execute("""SET work_mem TO '256MB';""")
  103. start_time = time.time()
  104. cursor.execute("""select sum(dn_2017) from grid_centroid_us_2017 where ST_Intersects(geom, ST_Buffer(ST_Transform(ST_SetSRID('{}'::geometry, 4326),102003),100));""".format(row[1]))
  105. result = cursor.fetchall()
  106. end = time.time()
  107. time_taken = end - start_time
  108. for i in result:
  109. row = [row[0], row[1], i[0], start_time, end, time_taken]
  110. outfile_writer.writerow(row)
  111. conn.close()
  112. outfile.close()
  113.  
  114.  
  115. if __name__ == "__main__":
  116. tr = TemporalRisk()
  117.  
  118. print("Reading data....")
  119. input_data = tr.read_data()
  120.  
  121. print("Processing data....")
  122. con = tr.get_connection()
  123. res = tr.process_data(con, input_data)
  124.  
  125. print("Writing data...")
  126. tr.write_data(res)
  127. print("Completed.....")
Add Comment
Please, Sign In to add comment