Advertisement
Guest User

Untitled

a guest
Jun 23rd, 2017
94
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 7.45 KB | None | 0 0
  1. #!/usr/bin/env python
  2. # -*- coding: utf-8 -*-
  3.  
  4. import os
  5. import csv
  6. import psycopg2
  7. import datetime
  8. from timeit import default_timer as timer
  9.  
  10. # Connection details:
  11. dbname='db'
  12. user='user'
  13. host='host:port'
  14. password='pass'
  15.  
  16.  
  17. def get_tables_databases():
  18. with psycopg2.connect(dbname=dbname, user=user, host=host, password=password) as conn:
  19. with conn.cursor() as cur:
  20. cur.execute('select table_name from INFORMATION_SCHEMA.TABLES where table_type=\'BASE TABLE\' and table_schema not in (\'pg_catalog\', \'information_schema\');')
  21. return [r for (r,) in cur]
  22.  
  23.  
  24. def get_column_table(table_name: str):
  25. with psycopg2.connect(dbname=dbname, user=user, host=host, password=password) as conn:
  26. with conn.cursor() as cur:
  27. cur.execute('select column_name from INFORMATION_SCHEMA.COLUMNS where table_name=\'' + table_name + '\';')
  28. return [r for (r,) in cur]
  29.  
  30.  
  31. def get_column_details_table(table_name: str):
  32. with psycopg2.connect(dbname=dbname, user=user, host=host, password=password) as conn:
  33. with conn.cursor() as cur:
  34. cur.execute('select column_name, data_type, character_maximum_length from INFORMATION_SCHEMA.COLUMNS where table_name=\'' + table_name + '\';')
  35. return [r for r in cur]
  36.  
  37.  
  38. def get_length_table(table_name: str):
  39. with psycopg2.connect(dbname=dbname, user=user, host=host, password=password) as conn:
  40. with conn.cursor() as cur:
  41. cur.execute('select count(*) from "' + table_name + '";')
  42. return cur.fetchone()[0]
  43.  
  44.  
  45. def get_aprox_length_table(table_name: str):
  46. with psycopg2.connect(dbname=dbname, user=user, host=host, password=password) as conn:
  47. with conn.cursor() as cur:
  48. cur.execute('select reltuples as estimate from pg_class where relname=\'' + table_name + '\';')
  49. return cur.fetchone()[0]
  50.  
  51.  
  52. def get_size_database():
  53. with psycopg2.connect(dbname=dbname, user=user, host=host, password=password) as conn:
  54. with conn.cursor() as cur:
  55. cur.execute('select pg_size_pretty(pg_database_size(pg_database.datname)) as size from pg_database where pg_database.datname=\'' + dbname + '\';')
  56. return cur.fetchone()[0]
  57.  
  58.  
  59. def get_rows_table(table_name: str,):
  60. with psycopg2.connect(dbname=dbname, user=user, host=host, password=password) as conn:
  61. with conn.cursor() as cur:
  62. cur.execute('select * from ' + table_name + ';')
  63. return cur.fetchall()
  64.  
  65.  
  66. def export_table_csv(table_name: str, file_path: str = None, with_title: bool = False):
  67. file_name = os.path.join(file_path if file_path else '.', table_name + '.csv')
  68. os.makedirs(os.path.dirname(file_name), exist_ok=True)
  69. with open(file_name, mode="w", encoding="UTF-8") as csvfile:
  70. csvwriter = csv.writer(csvfile, delimiter=' ', quotechar='|', quoting=csv.QUOTE_MINIMAL)
  71. with psycopg2.connect(dbname=dbname, user=user, host=host, password=password) as conn:
  72. # column names
  73. if with_title:
  74. with conn.cursor() as cur:
  75. cur.execute('select column_name from INFORMATION_SCHEMA.COLUMNS where table_name=\'' + table_name + '\';')
  76. title = []
  77. for (r,) in cur:
  78. title.append(r)
  79. csvwriter.writerow(title)
  80. # content
  81. with conn.cursor('server-side-cursor') as cur:
  82. cur.itersize = 100000 # how much records to buffer on a client
  83. cur.execute('select * from ' + table_name + ';')
  84. for r in cur:
  85. csvwriter.writerow(r)
  86.  
  87.  
  88. def export_database_csv(file_path: str = None, with_title: bool = False):
  89. file_path = file_path if file_path else dbname
  90. os.makedirs(file_path, exist_ok=True)
  91. for table in get_tables_databases():
  92. export_table_csv(table, file_path, with_title)
  93.  
  94.  
  95. def export_table_copy(table_name: str, file_path: str = None, with_title: bool = False):
  96. file_name = os.path.join(file_path if file_path else '.', table_name + '.copy')
  97. os.makedirs(os.path.dirname(file_name), exist_ok=True)
  98. with open(file_name, mode="w", encoding="UTF-8") as copyfile:
  99. with psycopg2.connect(dbname=dbname, user=user, host=host, password=password) as conn:
  100. with conn.cursor() as cur:
  101. # column names
  102. if with_title:
  103. cur.execute('select column_name from INFORMATION_SCHEMA.COLUMNS where table_name=\'' + table_name + '\';')
  104. title = ''
  105. for (r,) in cur:
  106. title += r + '|'
  107. copyfile.write(title[:-1] + '\n')
  108. # content
  109. cur.copy_to(copyfile, table_name, sep="|")
  110.  
  111.  
  112. def export_database_copy(file_path: str = None, with_title: bool = False):
  113. file_path = file_path if file_path else dbname
  114. os.makedirs(file_path, exist_ok=True)
  115. for table in get_tables_databases():
  116. export_table_copy(table, file_path, with_title)
  117.  
  118.  
  119. def export_table_sql(table_name: str, file_path: str = None):
  120. file_name = os.path.join(file_path if file_path else '.', table_name + '.sql')
  121. os.makedirs(os.path.dirname(file_name), exist_ok=True)
  122. with open(file_name, mode="w", encoding="UTF-8") as sqlfile:
  123. with psycopg2.connect(dbname=dbname, user=user, host=host, password=password) as conn:
  124. # table
  125. with conn.cursor() as cur:
  126. cur.execute('select column_name, data_type, character_maximum_length from INFORMATION_SCHEMA.COLUMNS where table_name=\'' + table_name + '\';')
  127. columns = []
  128. for r in cur:
  129. (cname, ctype, clen) = r
  130. if ctype in {'character varying', 'varchar', 'character', 'char', 'text'}:
  131. ctype = 'varchar'
  132. elif ctype == 'money':
  133. ctype = 'double precision'
  134. elif ctype == 'bytea':
  135. ctype = 'BLOB'
  136. clen = '(' + clen + ')' if clen and ctype == 'varchar' else ''
  137. columns.append(cname + ' ' + ctype + clen)
  138. sqlfile.write('create table \'' + table_name + '\' (' + ', '.join(columns) + ');\n')
  139. # rows
  140. with conn.cursor('server-side-cursor') as cur:
  141. cur.itersize = 100000 # how much records to buffer on a client
  142. cur.execute('select * from ' + table_name + ';')
  143. for r in cur:
  144. data = []
  145. for rd in r:
  146. if rd == 'null':
  147. data.append('NULL')
  148. else:
  149. data.append(repr(rd))
  150. sqlfile.write('insert into ' + table_name + ' values (' + ', '.join(data) + ');\n')
  151.  
  152.  
  153. def export_database_sql(file_path: str = None):
  154. file_path = file_path if file_path else dbname
  155. os.makedirs(file_path, exist_ok=True)
  156. for table in get_tables_databases():
  157. export_table_sql(table, file_path)
  158.  
  159.  
  160. # test :
  161. print(get_tables_databases())
  162. print(get_size_database())
  163. print(get_column_table('a_table'))
  164. for col in get_column_details_table('a_table'):
  165. print(col)
  166. print(get_length_table('a_table'))
  167. print(get_aprox_length_table('a_table'))
  168. # export bench
  169. start = timer()
  170. export_table_sql('a_table')
  171. end = timer()
  172. print(end-start)
  173. # export bench
  174. start = timer()
  175. export_table_csv('a_table')
  176. end = timer()
  177. print(end-start)
  178. # export bench
  179. start = timer()
  180. export_table_copy('a_table')
  181. end = timer()
  182. print(end-start)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement