Guest User

Untitled

a guest
Jun 20th, 2018
98
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 4.09 KB | None | 0 0
  1. import os
  2. import psycopg2
  3. import subprocess
  4.  
  5.  
  6. def runogr2ogr(infile):
  7. # import the data
  8. try:
  9. command = ["ogr2ogr", "-f", "PostgreSQL",
  10. OGR_CONN_STRING,
  11. infile]
  12. print command
  13. subprocess.check_call(command)
  14. # record an error if there is one
  15. except subprocess.CalledProcessError as e:
  16. print(str(e.output))
  17.  
  18.  
  19. def runosmium(infile, outfile, month, year):
  20. month_run = month + 1
  21. if month_run == 13:
  22. month_run = 1
  23. year_run = year + 1
  24. else:
  25. year_run = year
  26. month_run = str(month_run).zfill(2)
  27. year_run = str(year_run).zfill(4)
  28.  
  29. try:
  30. command = ["osmium", "time-filter", infile,
  31. "{}-{}-01T00:00:00Z".format(year_run, month_run), "-o", outfile]
  32. print command
  33. subprocess.check_call(command)
  34. except subprocess.CalledProcessError as e:
  35. print(str(e.output))
  36.  
  37.  
  38. def checkTableExits(schema, table, connection):
  39. m_cur = connection.cursor()
  40. m_cur.execute(
  41. "select exists (select * from information_schema.tables where "
  42. "table_schema='{}' AND table_name='{}')".format(
  43. schema, table
  44. ))
  45. if m_cur.fetchone()[0]:
  46. ret = True
  47. else:
  48. ret = False
  49. m_cur.close()
  50. return ret
  51.  
  52.  
  53. # Dates
  54. START_YEAR = 2006
  55. START_MONTH = 12
  56. END_YEAR = 2018
  57. END_MONTH = 4
  58.  
  59. ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
  60. INFILE = "ireland-and-northern-ireland.osh.pbf"
  61.  
  62.  
  63. CONN_STRING = "dbname=%s user=%s password=%s host=%s port=%s" % (
  64. "postgis", "postgres", "postgres", "localhost", "5432")
  65.  
  66. OGR_CONN_STRING = "PG:host=localhost user=postgres password=postgres dbname=postgis port=5432"
  67. conn = psycopg2.connect(CONN_STRING)
  68.  
  69. SCHEMA = "public"
  70.  
  71. for y in range(START_YEAR, END_YEAR + 1):
  72. if y == START_YEAR and END_YEAR > START_YEAR:
  73. t_end_month = 12
  74. elif y > START_YEAR and y < END_YEAR:
  75. t_end_month = 12
  76. else:
  77. t_end_month = END_MONTH
  78. if y > START_YEAR:
  79. t_start_month = 01
  80. else:
  81. t_start_month = START_MONTH
  82. for m in range(t_start_month, t_end_month + 1):
  83.  
  84. print(y, m)
  85.  
  86. file_name_pbf = str(y) + str(m).zfill(2) + '.pbf'
  87. out_file_pbf = os.path.join(ROOT_DIR, file_name_pbf)
  88. runosmium(INFILE, out_file_pbf, m, y)
  89.  
  90. if checkTableExits(SCHEMA, 'public.lines', conn) is True:
  91. runogr2ogr(out_file_pbf)
  92. else:
  93. runogr2ogr(out_file_pbf)
  94. cur = conn.cursor()
  95. cur.execute("ALTER TABLE public.lines ADD COLUMN IF NOT EXISTS load_date date")
  96. conn.commit()
  97. cur.execute("ALTER TABLE public.multilinestrings ADD COLUMN IF NOT EXISTS load_date date")
  98. conn.commit()
  99. cur.execute("ALTER TABLE public.multipolygons ADD COLUMN IF NOT EXISTS load_date date")
  100. conn.commit()
  101. cur.execute("ALTER TABLE public.other_relations ADD COLUMN IF NOT EXISTS load_date date")
  102. conn.commit()
  103. cur.execute("ALTER TABLE public.points ADD COLUMN IF NOT EXISTS load_date date")
  104. conn.commit()
  105. cur.close()
  106.  
  107. cur = conn.cursor()
  108. cur.execute(
  109. "update public.lines set load_date = '{}-{}-01' where load_date is NULL;".format(
  110. str(y), str(m).zfill(2)))
  111. conn.commit()
  112. cur.execute(
  113. "update public.multilinestrings set load_date = '{}-{}-01' where load_date is NULL;".format(
  114. str(y), str(m).zfill(2)))
  115. conn.commit()
  116. cur.execute(
  117. "update public.multipolygons set load_date = '{}-{}-01' where load_date is NULL;".format(
  118. str(y), str(m).zfill(2)))
  119. conn.commit()
  120. cur.execute(
  121. "update public.other_relations set load_date = '{}-{}-01' where load_date is NULL;".format(
  122. str(y), str(m).zfill(2)))
  123. conn.commit()
  124. cur.execute(
  125. "update public.points set load_date = '{}-{}-01' where load_date is NULL;".format(
  126. str(y), str(m).zfill(2)))
  127. conn.commit()
  128. cur.close()
  129. os.remove(out_file_pbf)
Add Comment
Please, Sign In to add comment