Ticket #695: transition.diff

File transition.diff, 20.0 KB (added by brantley (deadwisdom@…, 14 years ago)

Transition patch

  • bin/django-admin.py

     
    66ACTION_MAPPING = {
    77    'adminindex': management.get_admin_index,
    88    'createsuperuser': management.createsuperuser,
     9    'transition': management.transition,
    910    'createcachetable' : management.createcachetable,
    1011#     'dbcheck': management.database_check,
    1112    'init': management.init,
     
    1819    'sqlindexes': management.get_sql_indexes,
    1920    'sqlinitialdata': management.get_sql_initial_data,
    2021    'sqlreset': management.get_sql_reset,
     22    'sqlupdate': management.get_sql_update,
    2123    'sqlsequencereset': management.get_sql_sequence_reset,
    2224    'startapp': management.startapp,
    2325    'startproject': management.startproject,
    2426    'validate': management.validate,
    2527}
    2628
    27 NO_SQL_TRANSACTION = ('adminindex', 'createcachetable', 'dbcheck', 'install', 'sqlindexes')
     29NO_SQL_TRANSACTION = ('adminindex', 'createcachetable', 'dbcheck', 'install', 'sqlindexes', 'transition')
    2830
    2931def get_usage():
    3032    """
  • core/meta/transition.py

     
     1from django.core import db, meta
     2
     3def create_column(f):
     4    "Create the sql for a specific column.  Took this straight from management.get_sql_create()"
     5    if isinstance(f, meta.ForeignKey):
     6        rel_field = f.rel.get_related_field()
     7        # If the foreign key points to an AutoField, the foreign key
     8        # should be an IntegerField, not an AutoField. Otherwise, the
     9        # foreign key should be the same type of field as the field
     10        # to which it points.
     11        if rel_field.__class__.__name__ == 'AutoField':
     12            data_type = 'IntegerField'
     13        else:
     14            data_type = rel_field.__class__.__name__
     15    else:
     16        rel_field = f
     17        data_type = f.__class__.__name__
     18    col_type = db.DATA_TYPES[data_type]
     19    if col_type is not None:
     20        field_output = ["`%s`" % f.column, col_type % rel_field.__dict__]
     21        field_output.append('%sNULL' % (not f.null and 'NOT ' or ''))
     22        if f.unique:
     23            field_output.append('UNIQUE')
     24        if f.primary_key:
     25            field_output.append('PRIMARY KEY')
     26        if f.rel:
     27            field_output.append('REFERENCES %s (%s)' % \
     28                (f.rel.to.db_table, f.rel.to.get_field(f.rel.field_name).column))
     29        return ' '.join(field_output)
     30    else:
     31        return None
     32
     33class TableState:
     34    "Holds state information about a table."
     35    table = None
     36    new_name = None
     37    drops = []
     38    creates = []
     39    changes = {}
     40   
     41    _database_state = None
     42   
     43    def __init__(self, table, database_state):
     44        self.table = table
     45        self._database_state = database_state
     46        self.new_name = None
     47        self.drops = []
     48        self.creates = []
     49        self.changes = {}
     50   
     51    def update(self, type, field, args):
     52        "Updates the current properties, according to the change-tuple data recieved"
     53        if (type == 'Name'):
     54            if (field == None):
     55                self.new_name = args[0]
     56                self._database_state.name_change(self.table, self.new_name)
     57            else:
     58                self.add_name_change(field, args[0])
     59        elif (type == 'Drop'):
     60            self.drops.append(field)
     61        elif (type == 'Add'):
     62            self.creates.append(field)
     63        elif (type == 'Change'):
     64            self.add_change(field)
     65        else:
     66            raise NameError("Unknown change type: %r" % type)
     67   
     68    def add_change(self, field):
     69        self.changes[field] = field
     70   
     71    def add_name_change(self, field, new_name):
     72        self.changes[field] = new_name
     73   
     74    def get_field(self, field_name, klass):
     75        "Finds a specific field in a klass, raises a NameError if it can't find it."
     76        for f in klass._meta.fields:
     77            if (f.name == field_name):
     78                return f
     79        raise NameError("Cannot find the specified field: %r" % field_name)
     80   
     81    def get_many_to_many(self, field_name, klass):
     82        "Finds a specific many to many field, or returns None if it can't find it."
     83        for f in klass._meta.many_to_many:
     84            if (f.name == field_name):
     85                return f
     86        return None
     87   
     88    def render_sql(self, mod):
     89        "Renders the sql for this specific table."
     90        sql = []
     91       
     92        table_name = self._database_state.get_table_name(self.table)
     93       
     94        if (self.new_name):
     95            new_name = self._database_state.get_table_name(self.new_name)
     96            self.table = self.new_name
     97            sql.append("RENAME TO `%s`" % new_name)
     98       
     99        for c in self.drops:
     100            sql.append("DROP COLUMN `%s`" % c)
     101       
     102        klass = self._database_state.get_module(self.table)
     103        for col in self.creates:
     104            try:
     105                field = self.get_field(col, klass)
     106            except NameError, e:
     107                field = self.get_many_to_many(col, klass)
     108                if (field):
     109                    self._database_state.add_m2m(field, klass)
     110                    continue
     111                else:
     112                    raise e
     113            col_text = create_column(field)
     114            if (col_text):
     115                sql.append("ADD COLUMN %s" % col_text)
     116            else:
     117                raise Exception("Data type unknown, could not create the column: %r" % col)
     118           
     119        for c in self.changes.keys():
     120            name = self.changes[c]
     121            col_text = create_column(self.get_field(name, klass))
     122            if (col_text):
     123                sql.append("CHANGE COLUMN `%s` %s" % (c, col_text))
     124            else:
     125                raise Exception("Data type unknown, could not create the column: %r" % c)
     126           
     127        return "ALTER TABLE `%s` %s;" % (table_name, ",\n".join(sql))
     128   
     129class DatabaseState:
     130    "Holds state information for a database."
     131   
     132    #There are four basic changes that can happen in a database:
     133    drops = []       #  Dropping a table
     134    creates = []   #  Creating a table
     135    alters = {}   #  Altering a table
     136    m2m = []     # Adding a Many-To-Many table
     137   
     138    _name_changes = {}  # Hold any name changes.  We want to make changes on a table all at once.
     139                                   # This makes sure that if we change the name, and then reference it later,
     140                                   # we'll be referencing the same table.
     141   
     142    def __init__(self, change_tuples):
     143        "DatabaseState is initialized by sending it the change_tuples taken from the transition file."
     144        self.drops = []
     145        self.creates = []
     146        self.alters = {}
     147        self.m2m = []
     148        self._name_changes = {}
     149        for c in change_tuples:
     150            type = c[0]
     151           
     152            #Find the target, if it's a table field, then it'll have the pattern: "table.field", otherwise it'll just be "name"
     153            target = c[1].split('.')
     154            field = None
     155            if len(target) > 1:
     156                field = target[1]
     157            table = target[0]
     158           
     159            #Presently the only arguments added are for name changes.
     160            args = ()
     161            if len(c) > 2:
     162                args = c[2:]
     163           
     164            if (field or type == 'Name'):
     165                self.add_alteration(type, table, field, args)
     166            elif (type == 'Drop'):
     167                self.add_drop(table)
     168            elif (type == 'Add'):
     169                self.add_create(table)
     170   
     171    def add_drop(self, table):
     172        table = self.resolve_name(table)
     173        if (not table in self.drops):
     174            self.drops.append(table)
     175   
     176    def add_create(self, table):
     177        table = self.resolve_name(table)
     178        if (not table in self.creates):
     179            self.creates.append(table)
     180   
     181    def add_alteration(self, type, table, field, args):
     182        table = self.resolve_name(table)
     183        if (not table in self.alters):
     184            self.alters[table] = TableState(table, self)
     185        self.alters[table].update(type, field, args)
     186   
     187    def name_change(self, table, new_name):
     188        "Marks when a name change has occurred."
     189        self._name_changes[new_name] = table
     190       
     191    def resolve_name(self, table):
     192        "Resolve a table name, in case it's name had been changed."
     193        if (table in self._name_changes):
     194            return self._name_changes[table]
     195        else:
     196            return table
     197   
     198    def get_table_name(self, table_name):
     199        "Gets the true table name in case we have a module name."
     200        for klass in self.mod._MODELS:
     201            if (table_name) == klass.__name__:
     202                return klass._meta.db_table
     203        return table_name    #It must be a name in the database, not in the models.
     204   
     205    def get_module(self, name):
     206        "Gets the module with the given name."
     207        for k in self.mod._MODELS:
     208            if (k.__name__ == name):
     209                return k
     210        raise NameError("Cannot find a module named: %r" % name)
     211   
     212    def add_m2m(self, field, klass):
     213        "Marks that a many-to-many field has been found and adds it to the list for future proccessing."
     214        self.m2m.append((field, klass))
     215   
     216    def create_m2m(self, f, klass):
     217        "Creates the sql for a many-to-many field."
     218        opts = klass._meta
     219        table_output = ['CREATE TABLE %s (' % f.get_m2m_db_table(opts)]
     220        table_output.append('    id %s NOT NULL PRIMARY KEY,' % db.DATA_TYPES['AutoField'])
     221        table_output.append('    %s_id %s NOT NULL REFERENCES %s (%s),' % \
     222            (opts.object_name.lower(), db.DATA_TYPES['IntegerField'], opts.db_table, opts.pk.column))
     223        table_output.append('    %s_id %s NOT NULL REFERENCES %s (%s),' % \
     224            (f.rel.to.object_name.lower(), db.DATA_TYPES['IntegerField'], f.rel.to.db_table, f.rel.to.pk.column))
     225        table_output.append('    UNIQUE (%s_id, %s_id)' % (opts.object_name.lower(), f.rel.to.object_name.lower()))
     226        table_output.append(');')
     227        return "\n".join(table_output)
     228   
     229    def create_table(self, klass_name):
     230        "Renders the sql to create a brand-new table.  Mostly comes from management.get_sql_create(), but I simplified some things."
     231        mod = self.mod
     232       
     233        klass = self.get_module(klass_name)
     234       
     235        opts = klass._meta
     236        table_output = []
     237        for f in opts.fields:
     238            col_text = create_column(f) # create_column() is defined at the top of this text.
     239            if (col_text):
     240                table_output.append(col_text)
     241        if opts.order_with_respect_to:
     242            table_output.append('_order %s NULL' % db.DATA_TYPES['IntegerField'])
     243        for field_constraints in opts.unique_together:
     244            table_output.append('UNIQUE (%s)' % ", ".join([opts.get_field(f).column for f in field_constraints]))
     245
     246        full_statement = ['CREATE TABLE %s (' % opts.db_table]
     247        for i, line in enumerate(table_output): # Combine and add commas.
     248            full_statement.append('    %s%s' % (line, i < len(table_output)-1 and ',' or ''))
     249        full_statement.append(');')
     250        return '\n'.join(full_statement)
     251   
     252    def render_sql(self, mod):
     253        "Renders the sql for the app transition."
     254        self.mod = mod
     255       
     256        sql = []
     257        for table in self.drops:
     258            table = self.get_table_name(table)
     259            sql.append("DROP TABLE `%s`;" % table)
     260       
     261        for table in self.creates:
     262            sql.append(self.create_table(table))
     263       
     264        for table in self.alters.keys():
     265            sql.append(self.alters[table].render_sql(mod))
     266           
     267        for field, klass in self.m2m:
     268            sql.append(self.create_m2m(field, klass))
     269       
     270        return "\n".join(sql)
     271
     272# Hold the directives from the transition file
     273_change_tuples = []
     274
     275def _update_change(*tuple):
     276    _change_tuples.append(tuple)
     277
     278def render_sql(mod):
     279    "Renders the sql for the app transition."
     280    d = DatabaseState(_change_tuples)
     281    return d.render_sql(mod)
     282   
     283# These are the transition directives available.
     284def Name(db_name, name):
     285    """Name change for a table or a field, the first argument is the
     286    current name (either in the database or in the model), and
     287    the second is the new name as seen in the model."""
     288    _update_change("Name", db_name, name)
     289
     290def Add(name):
     291    "Adds/Creates a model or model field for the database."
     292    _update_change("Add", name)
     293
     294def Drop(db_table):
     295    "Drops a database table or field from the database."
     296    _update_change("Drop", db_table)
     297
     298def Change(name):
     299    "Realize a change in a model field for the database."
     300    _update_change("Change", name)
     301 No newline at end of file
  • core/management.py

     
    264264            contenttypes_seen[row[0]]
    265265        except KeyError:
    266266#             sys.stderr.write("A content type called '%s.%s' was found in the database but not in the model.\n" % (app_label, row[0]))
    267             print "DELETE FROM content_types WHERE package='%s' AND python_module_name = '%s';" % (app_label, row[0])
     267            print "DELETE FROM content_types WHERE package='%s' AND python_module_name = '%s';" % (app_label, row[0])
     268   
     269    transition(mod)
    268270database_check.help_doc = "Checks that everything is installed in the database for the given model module name(s) and prints SQL statements if needed."
    269271database_check.args = APP_ARGS
     272
     273def _compare_lists(first, second):
     274    "Returns a percentage likeness of one list to the other, lists are treated as sets (no repeats)."
     275    number = 0.0
     276    same = 0.0
     277    for i in first:
     278        if i in second:
     279            same += 1.0
     280        number += 1.0
     281    return same / number
     282   
     283def _get_fields_from_database(cursor, table_name):
     284    cursor.execute("SELECT * FROM %s LIMIT 1" % table_name)
     285    return [row[0] for i, row in enumerate(cursor.description)]
     286
     287def transition(mod):
     288    from django.core import db, meta
     289   
     290    changes = []
     291   
     292    # Get all the tables in the db that are part of our app
     293    appname = mod._MODELS[0]._meta.app_label
     294    cursor = db.db.cursor()
     295    tables = db.get_table_list(cursor)
     296    def helper(x):
     297        return x.startswith(appname+"_")
     298    tables = filter(helper, tables)
     299   
     300    # Find the classes in our current model-file that don't have a presence in our database.
     301    create_candidates = []
     302    drop_candidates = tables
     303    for klass in mod._MODELS:
     304        table_name = klass._meta.db_table
     305        if (table_name in tables):
     306            drop_candidates.remove(table_name)
     307        else:
     308            create_candidates.append(klass)
     309   
     310    # Find the ManyToManyFields
     311    for klass in mod._MODELS:
     312        opts = klass._meta
     313        for f in opts.many_to_many:
     314            m = f.get_m2m_db_table(opts)
     315            if (m in drop_candidates):
     316                drop_candidates.remove(m)
     317            else:
     318                # It isn't in the database, so we need to add it
     319                changes.append(("Add", "%s.%s" %(klass.__name__, f.name)))  # Add Many2Many field and/or intermediate table
     320       
     321    # Decide if there was just a name change.  'drop_candidates' now has only the tables from the database that weren't matched up with an existing model-entity.
     322    for table in drop_candidates:
     323        field_names = _get_fields_from_database(cursor, table)
     324       
     325        # Figure out what the best match for this model that doesn't have a representation in the current database.
     326        best = (0, None)
     327        for klass in create_candidates:
     328            c_field_names = [f.column for f in klass._meta.fields]
     329     
     330            likeness = _compare_lists(field_names, c_field_names)
     331            if (likeness > best[0]):
     332                best = (likeness, klass)
     333               
     334        # If we have a best, add it to the changes.
     335        if (best[0]):
     336            changes.append(("Name", table, best[1].__name__))       # Change Table Name
     337            tables.remove(table)
     338            create_candidates.remove(best[1])
     339
     340    # Figure out which tables need to be added
     341    for c in create_candidates:
     342        changes.append(("Add", c.__name__))     # Add Table
     343       
     344    # Which ones need to be dropped
     345    for c in drop_candidates:
     346        changes.append(("Drop", c))                 # Drop Table
     347
     348    # Now let's look at the fields
     349    for klass in mod._MODELS:
     350        opts = klass._meta
     351        try:
     352            cursor.execute("SELECT * FROM %s LIMIT 1" % opts.db_table)
     353        except:
     354            continue
     355        db_fields = [row for i, row in enumerate(cursor.description)]
     356       
     357        field_drop_candidates = db_fields
     358        field_create_candidates = []
     359        for field in opts.fields:
     360            create = True
     361            for db_field in field_drop_candidates:
     362                if (db_field[0] == field.column):
     363                    # At this point I tried figuring out if the field needs to be changed, but it is so hard to get the metadata from the database that I decided to move on without it.
     364                    # The problem is that the database returns a number, which represents what you can find from FIELD_TYPES, but what to do with that?  It doesn't tell you
     365                    # What acutal *Field from Django to use.  You can't even find if the current one would fit in the database because the database returns all sorts of numbers
     366                    # that might or might not be what was originally put in the there by django.  If we held the meta-data in a seperate table then we could easily handle this.
     367                    field_drop_candidates.remove(db_field)
     368                    create = False
     369                    break
     370            if (create):
     371                field_create_candidates.append(field)
     372       
     373        for field in field_drop_candidates:
     374            changes.append(('Drop', "%s.%s" % (klass.__name__, field[0])))    #Add the field
     375           
     376        for field in field_create_candidates:
     377            changes.append(('Add', "%s.%s" % (klass.__name__, field.name))) #Drop the field
     378       
     379    # Setup the transition file
     380    file = mod.__file__
     381    path = file[:file.rfind('.')] + ".transition.py"
     382    print "Creating transition file at %r" % path
     383    file = open(path, 'w')
     384    file.write("""\
     385# Django transition file
     386from django.core.meta.transition import *
     387
     388""")
     389       
     390    for change in changes:
     391        esses = ", ".join((len(change) - 1) * ["%r"])
     392        s = "%%s(%s)\r\n" % esses
     393        file.write(s % change)
     394       
     395    file.close()
     396transition.help_doc = "Creates a transition file to go from the current database schema to your updated model."
     397transition.args = APP_ARGS
     398
     399def get_sql_update(mod):   
     400    appname = mod._MODELS[0]._meta.app_label
     401    file = mod.__file__
     402    path = file[:file.rfind('.')] + ".transition.py"
     403    if (os.path.exists(path)):
     404        #print "Using transition file at %r" % path
     405        pass
     406    else:
     407        raise AssertionError("You have to create a transition file before you sqlupdate.")
     408   
     409    locals = {}
     410    execfile(path, {}, locals)
     411    render_sql = locals["render_sql"]
     412    print render_sql(mod)
     413   
     414get_sql_update.help_doc = "Returns a list of the ALTER TABLE SQL statements for the given module."
     415get_sql_update.args = APP_ARGS
    270416
    271417def get_admin_index(mod):
    272418    "Returns admin-index template snippet (in list form) for the given module."
Back to Top