daily pastebin goal
46%
SHARE
TWEET

Untitled

a guest Oct 12th, 2017 48 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. import copy
  2.  
  3. from mygrations.formats.mysql.definitions.database import database
  4. from mygrations.formats.mysql.mygrations.operations.alter_table import alter_table
  5. from mygrations.formats.mysql.mygrations.operations.add_constraint import add_constraint
  6. from mygrations.formats.mysql.mygrations.operations.remove_table import remove_table
  7.  
  8. class mygration:
  9.     """ Creates a migration plan to update a database to a given spec.
  10.  
  11.     If only one database is passed in, it treats it as a structure to migrate
  12.     to starting from a blank slate, which primariliy means just figuring out
  13.     what order to add tables in to account for foreign key constraints.
  14.  
  15.     If two tables are present, then it will treat the second as the current
  16.     database constraint, and will figure out steps to execute in action
  17.     to update that second database to match the structure of the first.
  18.  
  19.     The general steps are:
  20.  
  21.     1. Check for foreign key errors: if the two database we are trying to migrate to doesn't support its own foreign key constraints, there is a show stopper
  22.     2. Loop through tables to be added, "creating" them as their foreign key constraints are filled, and ignoring any that have interlocking dependencies
  23.     3. Remove any FK constraints that are no longer used (done early to prevent trouble in the next step)
  24.     4. Modify tables as needed.  If modifying tables involves adding foreign key constraints that are not fulfilled, ignore those and add them later
  25.     5. See if we can add any remaining tables now that some table modifications have been applied
  26.     6. If there are still any outstanding tables to add, remove any foreign key constraints that are not fulfilled and add the tables without them
  27.     7. Apply all foreign key constraints that were previously ignored in steps 4 & 6
  28.     8. Remove any tables that need to be removed
  29.     """
  30.     def __init__(self, db_to, db_from = None):
  31.         """ Create a migration plan
  32.  
  33.         :param db_to: The target database structure to migrate to
  34.         :param db_from: The current database structure to migrate from
  35.         :type db_to: mygrations.formats.mysql.definitions.database
  36.         :type db_from: mygrations.formats.mysql.definitions.database
  37.         """
  38.  
  39.         self.db_to = db_to
  40.         self.db_from = db_from
  41.         [ self._errors_1215, self._operations ] = self._process()
  42.  
  43.     @property
  44.     def operations(self):
  45.         """ Public getter.  Returns list of operations to bring db_from to db_to
  46.  
  47.         If db_from doesn't exist then it will be a list of operations to
  48.         create db_to.
  49.  
  50.         :returns: A list of table operations
  51.         :rtype: [mygrations.formats.mysql.mygrations.operations.operation]
  52.         """
  53.         return self._operations
  54.  
  55.     @property
  56.     def errors_1215(self):
  57.         """ Public getter.  Returns list of 1215 errors (as strings)
  58.  
  59.         :returns: A list of 1215 error messages
  60.         :rtype: [string]
  61.         """
  62.         return self._errors_1215
  63.  
  64.     def __len__(self):
  65.         return len(self._operations)
  66.  
  67.     def __bool__(self):
  68.         return True if len(self._operations) else False
  69.  
  70.     def __str__( self ):
  71.         return "n".join([ str(x) for x in self._operations ])
  72.  
  73.     def __iter__(self):
  74.         return self._operations.__iter__()
  75.  
  76.     def _differences(self, a, b):
  77.         """
  78.         Calculates the difference between two OrderedDicts.
  79.  
  80.         https://codereview.stackexchange.com/a/176303/140581
  81.  
  82.         Duplication!!!! (formats.mysql.create_parser).  Sue me.
  83.  
  84.         :param a: OrderedDict
  85.         :param b: OrderedDict
  86.         :return: (added, removed, overlap)
  87.         """
  88.         return (
  89.             [key for key in b if key not in a],
  90.             [key for key in a if key not in b],
  91.             [key for key in a if key in b]
  92.         )
  93.  
  94.     def _process(self):
  95.         """ Figures out the operations (and proper order) need to get to self.db_to
  96.  
  97.         Excessively commented because there are a lot of details and this is a critical
  98.         part of the process
  99.         """
  100.  
  101.         # Our primary output is a list of operations, but there is more that we need
  102.         # to make all of this happen.  We need a database to keep track of the
  103.         # state of the database we are building after each operation is "applied"
  104.         tracking_db = copy.deepcopy(self.db_from) if self.db_from else database()
  105.  
  106.         # a little bit of extra processing will simplify our algorithm by a good chunk.
  107.         # The situation is much more complicated when we have a database we are migrating
  108.         # from, because columns might be added/removed/changed, and it might be (for instance)
  109.         # that the removal of a column breaks a foreign key constraint.  The general
  110.         # ambiguities introduced by changes happening in different times/ways makes it
  111.         # much more difficult to figure out when foreign key constraints can properly
  112.         # be added without triggering a 1215 error.  The simplest way to straighten this
  113.         # all out is to cheat: "mygrate" the "to" database all by itself.  Without a "from"
  114.         # the operations are more straight-forward, and we can figure out with less effort
  115.         # whether or not all FK constraints can be fulfilled.  If they aren't all fulfilled,
  116.         # then just quit now before we do anything.  If they are all fulfilled then we
  117.         # know our final table will be fine, so if we can just split off any uncertain
  118.         # foreign key constraints and apply them all at the end when our database is done
  119.         # being updated.  Simple(ish)!
  120.         if self.db_from:
  121.             check = mygration(self.db_to)
  122.             if check.errors_1215:
  123.                 return [ check.errors_1215, [] ]
  124.  
  125.         # First figure out the status of individual tables
  126.         db_from_tables = self.db_from.tables if self.db_from else {}
  127.         (tables_to_add, tables_to_remove, tables_to_update) = self._differences(db_from_tables, self.db_to.tables)
  128.  
  129.         # IMPORTANT! tracking db and tables_to_add are both passed in by reference
  130.         # (like everything in python), but in this case I actually modify them by reference.
  131.         # not my preference, but it makes it easier here
  132.         (errors_1215, operations) = self._process_adds(tracking_db, tables_to_add)
  133.  
  134.         # if we have errors we are done
  135.         if errors_1215:
  136.             return [ errors_1215, operations ]
  137.  
  138.         # now apply table updates.  This acts differently: it returns a dictionary with
  139.         # two sets of operations: one to update the tables themselves, and one to update
  140.         # the foreign keys.  The latter are applied after everything else has happened.
  141.         fk_operations = []
  142.         split_operations = self._process_updates(tracking_db, tables_to_update)
  143.         # remove fks first to avoid 1215 errors caused by trying to remove a column
  144.         # that is being used in a FK constraint that hasn't yet been removed
  145.         if split_operations['removed_fks']:
  146.             operations.extend(split_operations['removed_fks'])
  147.         if split_operations['kitchen_sink']:
  148.             operations.extend(split_operations['kitchen_sink'])
  149.         if split_operations['fks']:
  150.             fk_operations.extend(split_operations['fks'])
  151.  
  152.         # now that we got some tables modified let's try adding tables again
  153.         # if we have any left.  Remember that tracking_db and tables_to_add
  154.         # are modified in-place.  The point here is that there may be some
  155.         # tables to add that we were not able to add before because they
  156.         # relied on adding a column to a table before a foreign key could
  157.         # be supported.
  158.         if tables_to_add:
  159.             (errors_1215, more_operations) = self._process_adds(tracking_db, tables_to_add)
  160.             if more_operations:
  161.                 operations = operations.extend(more_operations)
  162.             if errors_1215:
  163.                 if fk_operations:
  164.                     operations.extend(fk_operations)
  165.                 retrun [ errors_1215, operations ]
  166.  
  167.         # At this point in time if we still have tables to add it is because
  168.         # they have mutually-dependent foreign key constraints.  The way to
  169.         # fix that is to be a bit smarter (but not too smart) and remove
  170.         # from the tables all foreign key constraints that can't be added yet.
  171.         # Then run the CREATE TABLE operations, and add the foreign key
  172.         # constraints afterward
  173.         for table_to_add in tables_to_add:
  174.             new_table = self.db_to.tables[table_to_add]
  175.             bad_constraints = tracking_db.unfulfilled_fks(new_table)
  176.             new_table_copy = copy.deepcopy(new_table)
  177.             create_fks = alter_table(table_to_add)
  178.             for constraint in bad_constraints.values():
  179.                 create_fks.add_operation(add_constraint(constraint['foreign_key']))
  180.                 new_table_copy.remove_constraint(constraint['foreign_key'])
  181.             operations.append(new_table_copy.create())
  182.             fk_operations.append(create_fks)
  183.  
  184.         # process any remaining fk constraints
  185.         if fk_operations:
  186.             operations.extend(fk_operations)
  187.  
  188.         # finally remove any tables
  189.         for table_to_remove in tables_to_remove:
  190.             operations.append(remove_table(table_to_remove))
  191.             tracking_db.remove_table(table_to_remove)
  192.  
  193.         # all done!!!
  194.         return [ errors_1215, operations ]
  195.  
  196.     def _process_adds(self, tracking_db, tables_to_add):
  197.         """ Runs through tables_to_add and resolves FK constraints to determine order to add tables in
  198.  
  199.         tracking_db and tables_to_add are passed in by reference and modified
  200.  
  201.         :returns: A list of 1215 error messages and a list of mygration operations
  202.         :rtype: ( [{'error': string, 'foreign_key': mygrations.formats.mysql.definitions.constraint}], [mygrations.formats.mysql.mygrations.operations.operation] )
  203.         """
  204.         errors_1215 = []
  205.         operations = []
  206.         good_tables = {}
  207.  
  208.         # keep looping through tables as long as we find some to process
  209.         # the while loop will stop under two conditions: if all tables
  210.         # are processed or if we stop adding tables, which happens if we
  211.         # have tables with mutualy-dependent foreign key constraints
  212.         last_number_to_add = 0
  213.         while tables_to_add and len(tables_to_add) != last_number_to_add:
  214.             last_number_to_add = len(tables_to_add)
  215.             for new_table_name in tables_to_add:
  216.                 new_table = self.db_to.tables[new_table_name]
  217.                 bad_constraints = tracking_db.unfulfilled_fks(new_table)
  218.  
  219.                 # if we found no problems then we can add this table to our
  220.                 # tracking db and add the "CREATE TABLE" operation to our list of operations
  221.                 if not bad_constraints:
  222.                     tables_to_add.remove(new_table_name)
  223.                     operations.append(new_table.create())
  224.                     tracking_db.add_table(new_table)
  225.                     continue
  226.  
  227.                 # the next question is whether this is a valid constraint
  228.                 # that simply can't be added yet (because it has dependencies
  229.                 # that have not been added) or if there is an actual problem
  230.                 # with the constraint.
  231.                 if new_table_name in good_tables:
  232.                     continue
  233.  
  234.                 # If we are here we have to decide if this table is fulfillable
  235.                 # eventually, or if there is a mistake with a foreign key that
  236.                 # we can't fix.  To tell the difference we just check if the
  237.                 # database we are migrating to can fulfill these foreign keys.
  238.                 broken_constraints = self.db_to.unfulfilled_fks(new_table)
  239.                 if not broken_constraints:
  240.                     good_tables[new_table_name] = True
  241.                     continue
  242.  
  243.                 # otherwise it is no good: record as such
  244.                 tables_to_add.remove(new_table_name)
  245.                 for error in broken_constraints.values():
  246.                     errors_1215.append(error['error'])
  247.  
  248.         return (errors_1215, operations)
  249.  
  250.     def _process_updates(self, tracking_db, tables_to_update):
  251.         """ Runs through tables_to_update and resolves FK constraints to determine order to add them in
  252.  
  253.         tracking_db is passed in by reference and modified
  254.  
  255.         This doesn't return a list of 1215 errors because those would have been
  256.         Taken care of the first run through when the "to" database was mygrated
  257.         by itself.  Instead, this separates alters and addition/modification of
  258.         foreign key updates into different operations so the foreign key updates
  259.         can be processed separately.
  260.  
  261.         :returns: a dict
  262.         :rtype: {'fks': list, 'kitchen_sink': list}
  263.         """
  264.  
  265.         tables_to_update = tables_to_update[:]
  266.  
  267.         operations = {
  268.             'removed_fks':  [],
  269.             'fks':          [],
  270.             'kitchen_sink': []
  271.         }
  272.  
  273.         for update_table_name in tables_to_update:
  274.             target_table = self.db_to.tables[update_table_name]
  275.             source_table = self.db_from.tables[update_table_name]
  276.  
  277.             more_operations = source_table.to(target_table, True)
  278.             if 'removed_fks' in more_operations:
  279.                 operations['removed_fks'].append( more_operations['removed_fks'] )
  280.                 for operation in more_operations['removed_fks']:
  281.                     tracking_db.apply_operation(update_table_name, operation)
  282.             if 'fks' in more_operations:
  283.                 operations['fks'].append(more_operations['fks'])
  284.                 for operation in more_operations['fks']:
  285.                     tracking_db.apply_operation(update_table_name, operation)
  286.             if 'kitchen_sink' in more_operations:
  287.                 operations['kitchen_sink'].append(more_operations['kitchen_sink'])
  288.                 for operation in more_operations['kitchen_sink']:
  289.                     tracking_db.apply_operation(update_table_name, operation)
  290.  
  291.         return operations
RAW Paste Data
Top