Ticket #695: transition.diff

bin/django-admin.py

@@ -6 +6 @@
 ACTION_MAPPING = {
     'adminindex': management.get_admin_index,
     'createsuperuser': management.createsuperuser,
+    'transition': management.transition,
     'createcachetable' : management.createcachetable,
 #     'dbcheck': management.database_check,
     'init': management.init,
@@ -18 +19 @@
     'sqlindexes': management.get_sql_indexes,
     'sqlinitialdata': management.get_sql_initial_data,
     'sqlreset': management.get_sql_reset,
+    'sqlupdate': management.get_sql_update,
     'sqlsequencereset': management.get_sql_sequence_reset,
     'startapp': management.startapp,
     'startproject': management.startproject,
     'validate': management.validate,
 }
 
-NO_SQL_TRANSACTION = ('adminindex', 'createcachetable', 'dbcheck', 'install', 'sqlindexes')
+NO_SQL_TRANSACTION = ('adminindex', 'createcachetable', 'dbcheck', 'install', 'sqlindexes', 'transition')
 
 def get_usage():
     """

core/meta/transition.py

@@ +1 @@
+from django.core import db, meta
+
+def create_column(f):
+    "Create the sql for a specific column.  Took this straight from management.get_sql_create()"
+    if isinstance(f, meta.ForeignKey):
+        rel_field = f.rel.get_related_field()
+        # If the foreign key points to an AutoField, the foreign key
+        # should be an IntegerField, not an AutoField. Otherwise, the
+        # foreign key should be the same type of field as the field
+        # to which it points.
+        if rel_field.__class__.__name__ == 'AutoField':
+            data_type = 'IntegerField'
+        else:
+            data_type = rel_field.__class__.__name__
+    else:
+        rel_field = f
+        data_type = f.__class__.__name__
+    col_type = db.DATA_TYPES[data_type]
+    if col_type is not None:
+        field_output = ["`%s`" % f.column, col_type % rel_field.__dict__]
+        field_output.append('%sNULL' % (not f.null and 'NOT ' or ''))
+        if f.unique:
+            field_output.append('UNIQUE')
+        if f.primary_key:
+            field_output.append('PRIMARY KEY')
+        if f.rel:
+            field_output.append('REFERENCES %s (%s)' % \
+                (f.rel.to.db_table, f.rel.to.get_field(f.rel.field_name).column))
+        return ' '.join(field_output)
+    else:
+        return None
+
+class TableState:
+    "Holds state information about a table."
+    table = None
+    new_name = None
+    drops = []
+    creates = []
+    changes = {}
+    
+    _database_state = None
+    
+    def __init__(self, table, database_state):
+        self.table = table
+        self._database_state = database_state
+        self.new_name = None
+        self.drops = []
+        self.creates = []
+        self.changes = {}
+    
+    def update(self, type, field, args):
+        "Updates the current properties, according to the change-tuple data recieved"
+        if (type == 'Name'):
+            if (field == None):
+                self.new_name = args[0]
+                self._database_state.name_change(self.table, self.new_name)
+            else:
+                self.add_name_change(field, args[0])
+        elif (type == 'Drop'):
+            self.drops.append(field)
+        elif (type == 'Add'):
+            self.creates.append(field)
+        elif (type == 'Change'):
+            self.add_change(field)
+        else:
+            raise NameError("Unknown change type: %r" % type)
+    
+    def add_change(self, field):
+        self.changes[field] = field
+    
+    def add_name_change(self, field, new_name):
+        self.changes[field] = new_name
+    
+    def get_field(self, field_name, klass):
+        "Finds a specific field in a klass, raises a NameError if it can't find it."
+        for f in klass._meta.fields:
+            if (f.name == field_name):
+                return f
+        raise NameError("Cannot find the specified field: %r" % field_name)
+    
+    def get_many_to_many(self, field_name, klass):
+        "Finds a specific many to many field, or returns None if it can't find it."
+        for f in klass._meta.many_to_many:
+            if (f.name == field_name):
+                return f
+        return None
+    
+    def render_sql(self, mod):
+        "Renders the sql for this specific table."
+        sql = []
+        
+        table_name = self._database_state.get_table_name(self.table)
+        
+        if (self.new_name):
+            new_name = self._database_state.get_table_name(self.new_name)
+            self.table = self.new_name
+            sql.append("RENAME TO `%s`" % new_name)
+        
+        for c in self.drops:
+            sql.append("DROP COLUMN `%s`" % c)
+        
+        klass = self._database_state.get_module(self.table)
+        for col in self.creates:
+            try:
+                field = self.get_field(col, klass)
+            except NameError, e:
+                field = self.get_many_to_many(col, klass)
+                if (field):
+                    self._database_state.add_m2m(field, klass)
+                    continue
+                else:
+                    raise e
+            col_text = create_column(field)
+            if (col_text):
+                sql.append("ADD COLUMN %s" % col_text)
+            else:
+                raise Exception("Data type unknown, could not create the column: %r" % col)
+            
+        for c in self.changes.keys():
+            name = self.changes[c]
+            col_text = create_column(self.get_field(name, klass))
+            if (col_text):
+                sql.append("CHANGE COLUMN `%s` %s" % (c, col_text))
+            else:
+                raise Exception("Data type unknown, could not create the column: %r" % c)
+            
+        return "ALTER TABLE `%s` %s;" % (table_name, ",\n".join(sql))
+    
+class DatabaseState:
+    "Holds state information for a database."
+    
+    #There are four basic changes that can happen in a database:
+    drops = []       #  Dropping a table
+    creates = []   #  Creating a table
+    alters = {}   #  Altering a table
+    m2m = []     # Adding a Many-To-Many table
+    
+    _name_changes = {}  # Hold any name changes.  We want to make changes on a table all at once.
+                                   # This makes sure that if we change the name, and then reference it later,
+                                   # we'll be referencing the same table.
+    
+    def __init__(self, change_tuples):
+        "DatabaseState is initialized by sending it the change_tuples taken from the transition file."
+        self.drops = []
+        self.creates = []
+        self.alters = {}
+        self.m2m = []
+        self._name_changes = {}
+        for c in change_tuples:
+            type = c[0]
+            
+            #Find the target, if it's a table field, then it'll have the pattern: "table.field", otherwise it'll just be "name"
+            target = c[1].split('.')
+            field = None
+            if len(target) > 1:
+                field = target[1]
+            table = target[0]
+            
+            #Presently the only arguments added are for name changes.
+            args = ()
+            if len(c) > 2:
+                args = c[2:]
+            
+            if (field or type == 'Name'):
+                self.add_alteration(type, table, field, args)
+            elif (type == 'Drop'):
+                self.add_drop(table)
+            elif (type == 'Add'):
+                self.add_create(table)
+    
+    def add_drop(self, table):
+        table = self.resolve_name(table)
+        if (not table in self.drops):
+            self.drops.append(table)
+    
+    def add_create(self, table):
+        table = self.resolve_name(table)
+        if (not table in self.creates):
+            self.creates.append(table)
+    
+    def add_alteration(self, type, table, field, args):
+        table = self.resolve_name(table)
+        if (not table in self.alters):
+            self.alters[table] = TableState(table, self)
+        self.alters[table].update(type, field, args)
+    
+    def name_change(self, table, new_name):
+        "Marks when a name change has occurred."
+        self._name_changes[new_name] = table
+        
+    def resolve_name(self, table):
+        "Resolve a table name, in case it's name had been changed."
+        if (table in self._name_changes):
+            return self._name_changes[table]
+        else:
+            return table
+    
+    def get_table_name(self, table_name):
+        "Gets the true table name in case we have a module name."
+        for klass in self.mod._MODELS:
+            if (table_name) == klass.__name__:
+                return klass._meta.db_table
+        return table_name    #It must be a name in the database, not in the models.
+    
+    def get_module(self, name):
+        "Gets the module with the given name."
+        for k in self.mod._MODELS:
+            if (k.__name__ == name):
+                return k
+        raise NameError("Cannot find a module named: %r" % name)
+    
+    def add_m2m(self, field, klass):
+        "Marks that a many-to-many field has been found and adds it to the list for future proccessing."
+        self.m2m.append((field, klass))
+    
+    def create_m2m(self, f, klass):
+        "Creates the sql for a many-to-many field."
+        opts = klass._meta
+        table_output = ['CREATE TABLE %s (' % f.get_m2m_db_table(opts)]
+        table_output.append('    id %s NOT NULL PRIMARY KEY,' % db.DATA_TYPES['AutoField'])
+        table_output.append('    %s_id %s NOT NULL REFERENCES %s (%s),' % \
+            (opts.object_name.lower(), db.DATA_TYPES['IntegerField'], opts.db_table, opts.pk.column))
+        table_output.append('    %s_id %s NOT NULL REFERENCES %s (%s),' % \
+            (f.rel.to.object_name.lower(), db.DATA_TYPES['IntegerField'], f.rel.to.db_table, f.rel.to.pk.column))
+        table_output.append('    UNIQUE (%s_id, %s_id)' % (opts.object_name.lower(), f.rel.to.object_name.lower()))
+        table_output.append(');')
+        return "\n".join(table_output)
+    
+    def create_table(self, klass_name):
+        "Renders the sql to create a brand-new table.  Mostly comes from management.get_sql_create(), but I simplified some things."
+        mod = self.mod
+        
+        klass = self.get_module(klass_name)
+        
+        opts = klass._meta
+        table_output = []
+        for f in opts.fields:
+            col_text = create_column(f) # create_column() is defined at the top of this text.
+            if (col_text):
+                table_output.append(col_text)
+        if opts.order_with_respect_to:
+            table_output.append('_order %s NULL' % db.DATA_TYPES['IntegerField'])
+        for field_constraints in opts.unique_together:
+            table_output.append('UNIQUE (%s)' % ", ".join([opts.get_field(f).column for f in field_constraints]))
+
+        full_statement = ['CREATE TABLE %s (' % opts.db_table]
+        for i, line in enumerate(table_output): # Combine and add commas.
+            full_statement.append('    %s%s' % (line, i < len(table_output)-1 and ',' or ''))
+        full_statement.append(');')
+        return '\n'.join(full_statement)
+    
+    def render_sql(self, mod):
+        "Renders the sql for the app transition."
+        self.mod = mod
+        
+        sql = []
+        for table in self.drops:
+            table = self.get_table_name(table)
+            sql.append("DROP TABLE `%s`;" % table)
+        
+        for table in self.creates:
+            sql.append(self.create_table(table))
+        
+        for table in self.alters.keys():
+            sql.append(self.alters[table].render_sql(mod))
+            
+        for field, klass in self.m2m:
+            sql.append(self.create_m2m(field, klass))
+        
+        return "\n".join(sql)
+
+# Hold the directives from the transition file
+_change_tuples = []
+
+def _update_change(*tuple):
+    _change_tuples.append(tuple)
+
+def render_sql(mod):
+    "Renders the sql for the app transition."
+    d = DatabaseState(_change_tuples)
+    return d.render_sql(mod)
+    
+# These are the transition directives available.
+def Name(db_name, name):
+    """Name change for a table or a field, the first argument is the 
+    current name (either in the database or in the model), and 
+    the second is the new name as seen in the model."""
+    _update_change("Name", db_name, name)
+
+def Add(name):
+    "Adds/Creates a model or model field for the database."
+    _update_change("Add", name)
+
+def Drop(db_table):
+    "Drops a database table or field from the database."
+    _update_change("Drop", db_table)
+
+def Change(name):
+    "Realize a change in a model field for the database."
+    _update_change("Change", name)
+ No newline at end of file

core/management.py

@@ -264 +264 @@
             contenttypes_seen[row[0]]
         except KeyError:
 #             sys.stderr.write("A content type called '%s.%s' was found in the database but not in the model.\n" % (app_label, row[0]))
-            print "DELETE FROM content_types WHERE package='%s' AND python_module_name = '%s';" % (app_label, row[0])
+            print "DELETE FROM content_types WHERE package='%s' AND python_module_name = '%s';" % (app_label, row[0])
+    
+    transition(mod)
 database_check.help_doc = "Checks that everything is installed in the database for the given model module name(s) and prints SQL statements if needed."
 database_check.args = APP_ARGS
+
+def _compare_lists(first, second):
+    "Returns a percentage likeness of one list to the other, lists are treated as sets (no repeats)."
+    number = 0.0
+    same = 0.0
+    for i in first:
+        if i in second:
+            same += 1.0
+        number += 1.0
+    return same / number
+    
+def _get_fields_from_database(cursor, table_name):
+    cursor.execute("SELECT * FROM %s LIMIT 1" % table_name)
+    return [row[0] for i, row in enumerate(cursor.description)]
+
+def transition(mod):
+    from django.core import db, meta
+    
+    changes = []
+    
+    # Get all the tables in the db that are part of our app
+    appname = mod._MODELS[0]._meta.app_label
+    cursor = db.db.cursor()
+    tables = db.get_table_list(cursor)
+    def helper(x):
+        return x.startswith(appname+"_")
+    tables = filter(helper, tables)
+    
+    # Find the classes in our current model-file that don't have a presence in our database.
+    create_candidates = []
+    drop_candidates = tables
+    for klass in mod._MODELS:
+        table_name = klass._meta.db_table
+        if (table_name in tables):
+            drop_candidates.remove(table_name)
+        else:
+            create_candidates.append(klass)
+    
+    # Find the ManyToManyFields
+    for klass in mod._MODELS:
+        opts = klass._meta
+        for f in opts.many_to_many:
+            m = f.get_m2m_db_table(opts)
+            if (m in drop_candidates):
+                drop_candidates.remove(m)
+            else:
+                # It isn't in the database, so we need to add it
+                changes.append(("Add", "%s.%s" %(klass.__name__, f.name)))  # Add Many2Many field and/or intermediate table
+        
+    # Decide if there was just a name change.  'drop_candidates' now has only the tables from the database that weren't matched up with an existing model-entity.
+    for table in drop_candidates:
+        field_names = _get_fields_from_database(cursor, table)
+        
+        # Figure out what the best match for this model that doesn't have a representation in the current database.
+        best = (0, None)
+        for klass in create_candidates:
+            c_field_names = [f.column for f in klass._meta.fields]
+     
+            likeness = _compare_lists(field_names, c_field_names)
+            if (likeness > best[0]):
+                best = (likeness, klass)
+                
+        # If we have a best, add it to the changes.
+        if (best[0]):
+            changes.append(("Name", table, best[1].__name__))       # Change Table Name
+            tables.remove(table)
+            create_candidates.remove(best[1])
+
+    # Figure out which tables need to be added
+    for c in create_candidates:
+        changes.append(("Add", c.__name__))     # Add Table
+        
+    # Which ones need to be dropped
+    for c in drop_candidates:
+        changes.append(("Drop", c))                 # Drop Table
+
+    # Now let's look at the fields
+    for klass in mod._MODELS:
+        opts = klass._meta
+        try:
+            cursor.execute("SELECT * FROM %s LIMIT 1" % opts.db_table)
+        except:
+            continue
+        db_fields = [row for i, row in enumerate(cursor.description)]
+        
+        field_drop_candidates = db_fields
+        field_create_candidates = []
+        for field in opts.fields:
+            create = True
+            for db_field in field_drop_candidates:
+                if (db_field[0] == field.column):
+                    # At this point I tried figuring out if the field needs to be changed, but it is so hard to get the metadata from the database that I decided to move on without it.
+                    # The problem is that the database returns a number, which represents what you can find from FIELD_TYPES, but what to do with that?  It doesn't tell you
+                    # What acutal *Field from Django to use.  You can't even find if the current one would fit in the database because the database returns all sorts of numbers
+                    # that might or might not be what was originally put in the there by django.  If we held the meta-data in a seperate table then we could easily handle this.
+                    field_drop_candidates.remove(db_field)
+                    create = False
+                    break
+            if (create):
+                field_create_candidates.append(field)
+        
+        for field in field_drop_candidates:
+            changes.append(('Drop', "%s.%s" % (klass.__name__, field[0])))    #Add the field
+            
+        for field in field_create_candidates:
+            changes.append(('Add', "%s.%s" % (klass.__name__, field.name))) #Drop the field
+        
+    # Setup the transition file
+    file = mod.__file__
+    path = file[:file.rfind('.')] + ".transition.py"
+    print "Creating transition file at %r" % path
+    file = open(path, 'w')
+    file.write("""\
+# Django transition file
+from django.core.meta.transition import *
+
+""")
+        
+    for change in changes:
+        esses = ", ".join((len(change) - 1) * ["%r"])
+        s = "%%s(%s)\r\n" % esses
+        file.write(s % change)
+        
+    file.close()
+transition.help_doc = "Creates a transition file to go from the current database schema to your updated model."
+transition.args = APP_ARGS
+
+def get_sql_update(mod):    
+    appname = mod._MODELS[0]._meta.app_label
+    file = mod.__file__
+    path = file[:file.rfind('.')] + ".transition.py"
+    if (os.path.exists(path)):
+        #print "Using transition file at %r" % path
+        pass
+    else:
+        raise AssertionError("You have to create a transition file before you sqlupdate.")
+    
+    locals = {}
+    execfile(path, {}, locals)
+    render_sql = locals["render_sql"]
+    print render_sql(mod)
+    
+get_sql_update.help_doc = "Returns a list of the ALTER TABLE SQL statements for the given module."
+get_sql_update.args = APP_ARGS
 
 def get_admin_index(mod):
     "Returns admin-index template snippet (in list form) for the given module."