Advertisement
Guest User

Untitled

a guest
Mar 20th, 2019
77
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 3.23 KB | None | 0 0
  1. # Drop child tables of intermediate aggregation tables.
  2. # Each table must have a 'month' column.
  3. # If tables have usage as reported by pg_stat_user_tables
  4. # they will not be dropped.
  5.  
  6. import itertools
  7. from datetime import date
  8. from collections import defaultdict
  9. from corehq.sql_db.connections import connection_manager
  10. from sqlalchemy.exc import ProgrammingError
  11. from six.moves import input
  12.  
  13.  
  14. def get_child_tables(engine, parent_table):
  15. with engine.begin() as conn:
  16. res = conn.execute(
  17. """
  18. SELECT c.relname AS child
  19. FROM
  20. pg_inherits JOIN pg_class AS c ON (inhrelid=c.oid)
  21. JOIN pg_class as p ON (inhparent=p.oid)
  22. where p.relname = %s;
  23. """,
  24. parent_table
  25. )
  26. return [row.child for row in res]
  27.  
  28.  
  29. def categorise_tables_by_date(engine, tables):
  30. tables_by_date = defaultdict(list)
  31. with engine.begin() as c:
  32. for t in tables:
  33. res = list(c.execute("select month from {} limit 1".format(t)))
  34. if res:
  35. tables_by_date[res[0].month].append(t)
  36. return tables_by_date
  37.  
  38.  
  39. def unlink_tables_from_parent(engine, parent_table, tables):
  40. tables_in_use = []
  41. unlinked = []
  42. not_exists = []
  43. for table in tables:
  44. try:
  45. with engine.begin() as conn:
  46. seq = conn.execute('select seq_tup_read from pg_stat_user_tables where relname = %s', table)
  47. reads = list(seq)[0].seq_tup_read
  48. if reads > 1: # we just read from the table so this won't be 0
  49. tables_in_use.append((table, reads))
  50. continue
  51. conn.execute('alter table "{}" no inherit "{}"'.format(table, parent_table))
  52. unlinked.append(table)
  53. except ProgrammingError as ex:
  54. if 'does not exist' not in str(ex):
  55. raise
  56. not_exists.append(table)
  57.  
  58. return unlinked, tables_in_use, not_exists
  59.  
  60.  
  61. def drop_tables(engine, tables):
  62. for table in tables:
  63. with engine.begin() as conn:
  64. conn.execute('drop table if exists "{}"'.format(table))
  65.  
  66.  
  67. def prune_child_tables(engine, parent_table, till_date):
  68. child_tables = get_child_tables(engine, parent_table)
  69. tables_by_date = categorise_tables_by_date(engine, child_tables)
  70. tables_to_prune = {
  71. date: tables
  72. for date, tables in tables_by_date.items()
  73. if date < till_date
  74. }
  75.  
  76. tables_in_order = itertools.chain(*[
  77. tables_to_prune[month]
  78. for month in sorted(tables_to_prune.keys())
  79. ])
  80. unlinked, tables_in_use, not_exists = unlink_tables_from_parent(engine, parent_table, tables_in_order)
  81. if tables_in_use:
  82. print("Some tables still in use:")
  83. for table, reads in tables_in_use:
  84. print("\t{} ({} reads)".format(table, reads))
  85. if not_exists:
  86. print("Tables that don't exist:")
  87. for table in not_exists:
  88. print("\t{}".format(table))
  89.  
  90. ok = input("Type 'drop' to continue: ")
  91. if ok != "drop":
  92. print('Abort')
  93. return
  94.  
  95. drop_tables(engine, unlinked)
  96.  
  97.  
  98. engine = connection_manager.get_engine('<engine ID>')
  99. parent_table = '<parent table>'
  100. prune_child_tables(engine, parent_table, date(2018, 6, 1))
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement