Changeset 5821 for django/branches/schema-evolution

Timestamp:

08/06/07 09:19:27 (1 year ago)

Author:

danderson

Message:

schema-evolution:
added support for custom migration scripts

Files:

• django/branches/schema-evolution/django/core/management.py (modified) (6 diffs)
• django/branches/schema-evolution/django/db/backends/mysql/introspection.py (modified) (1 diff)
• django/branches/schema-evolution/docs/schema-evolution.txt (modified) (6 diffs)

django/branches/schema-evolution/django/core/management.py

@@ -483 +483 @@
-    return output
-    
+def get_sql_fingerprint(app):
+    "Returns the fingerprint of the current schema, used in schema evolution."
+    from django.db import get_creation_module, models, backend, get_introspection_module, connection
+    # This should work even if a connecton isn't available
+    try:
+        cursor = connection.cursor()
+    except:
+        cursor = None
+    introspection = get_introspection_module()
+    app_name = app.__name__.split('.')[-2]
+    schema_fingerprint = introspection.get_schema_fingerprint(cursor, app)
+    try:
+        # is this a schema we recognize?
+        app_se = __import__(app_name +'.schema_evolution').schema_evolution
+        schema_recognized = schema_fingerprint in app_se.fingerprints
+        if schema_recognized:
+            sys.stderr.write(style.NOTICE("Notice: Current schema fingerprint for '%s' is '%s' (recognized)\n" % (app_name, schema_fingerprint)))
+        else:
+            sys.stderr.write(style.NOTICE("Notice: Current schema fingerprint for '%s' is '%s' (unrecognized)\n" % (app_name, schema_fingerprint)))
+    except:
+        sys.stderr.write(style.NOTICE("Notice: Current schema fingerprint for '%s' is '%s' (no schema_evolution module found)\n" % (app_name, schema_fingerprint)))
+    return
+get_sql_fingerprint.help_doc = "Returns the fingerprint of the current schema, used in schema evolution."
+get_sql_fingerprint.args = APP_ARGS
+
-def get_sql_evolution(app):
+    "Returns SQL to update an existing schema to match the existing models."
+    return get_sql_evolution_detailed(app)[2]
+
+def get_sql_evolution_detailed(app):
-    "Returns SQL to update an existing schema to match the existing models."
-    import schema_evolution
@@ -508 +537 @@
-    _check_for_validation_errors()
-
+    # This should work even if a connecton isn't available
+    try:
+        cursor = connection.cursor()
+    except:
+        cursor = None
+
+    introspection = get_introspection_module()
+    app_name = app.__name__.split('.')[-2]
+
-    final_output = []
+
+    schema_fingerprint = introspection.get_schema_fingerprint(cursor, app)
+    try:
+        # is this a schema we recognize?
+        app_se = __import__(app_name +'.schema_evolution').schema_evolution
+        schema_recognized = schema_fingerprint in app_se.fingerprints
+        if schema_recognized:
+            sys.stderr.write(style.NOTICE("Notice: Current schema fingerprint for '%s' is '%s' (recognized)\n" % (app_name, schema_fingerprint)))
+            available_upgrades = []
+            for (vfrom, vto), upgrade in app_se.evolutions.iteritems():
+                if vfrom == schema_fingerprint:
+                    try:
+                        distance = app_se.fingerprints.index(vto)-app_se.fingerprints.index(vfrom)
+                        available_upgrades.append( ( vfrom, vto, upgrade, distance ) )
+                        sys.stderr.write(style.NOTICE("\tan upgrade from %s to %s is available (distance: %i)\n" % ( vfrom, vto, distance )))
+                    except:
+                        available_upgrades.append( ( vfrom, vto, upgrade, -1 ) )
+                        sys.stderr.write(style.NOTICE("\tan upgrade from %s to %s is available, but %s is not in schema_evolution.fingerprints\n" % ( vfrom, vto, vto )))
+            if len(available_upgrades):
+                best_upgrade = available_upgrades[0]
+                for an_upgrade in available_upgrades:
+                    if an_upgrade[3] > best_upgrade[3]:
+                        best_upgrade = an_upgrade
+                final_output.extend( best_upgrade[2] )
+                return schema_fingerprint, False, final_output
+        else:
+            sys.stderr.write(style.NOTICE("Notice: Current schema fingerprint for '%s' is '%s' (unrecognized)\n" % (app_name, schema_fingerprint)))
+    except:
+        # sys.stderr.write(style.NOTICE("Notice: Current schema fingerprint for '%s' is '%s' (no schema_evolution module found)\n" % (app_name, schema_fingerprint)))
+        pass # ^^^ lets not be chatty
-
-    # stolen and trimmed from syncdb so that we know which models are about 
@@ -526 +594 @@
-        created_models.add(model)
-        table_list.append(model._meta.db_table)
-
-    introspection = get_introspection_module()
-    # This should work even if a connecton isn't available
-    try:
-        cursor = connection.cursor()
-    except:
-        cursor = None
-
-    # get the existing models, minus the models we've just created
@@ -557 +618 @@
-        final_output.extend(output)
-        
-
+schema_fingerprint, True, 
-    
-get_sql_evolution.help_doc = "Returns SQL to update an existing schema to match the existing models."
@@ -649 +710 @@
-                        cursor.execute(statement)
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
-
-    transaction.commit_unless_managed()
@@ -1603 +1673 @@
-    'sqlsequencereset': get_sql_sequence_reset,
-    'sqlevolve': get_sql_evolution,
+    'sqlfingerprint': get_sql_fingerprint,
-    'startapp': startapp,
-    'startproject': startproject,

django/branches/schema-evolution/django/db/backends/mysql/introspection.py

@@ -110 +110 @@
-    # print table_name, column_name, dict
-    return dict
+
+def get_schema_fingerprint(cursor, app):
+    """it's important that the output of these methods don't change, otherwise the hashes they
+    produce will be inconsistent (and detection of existing schemas will fail.  unless you are 
+    absolutely sure the outout for ALL valid inputs will remain the same, you should bump the version by creating a new method"""
+    return get_schema_fingerprint_fv1(cursor, app)
+
+def get_schema_fingerprint_fv1(cursor, app):
+    from django.db import models
+    app_name = app.__name__.split('.')[-2]
+
+    schema = ['app_name := '+ app_name]
+
+    cursor.execute('SHOW TABLES;')
+    for table_name in [row[0] for row in cursor.fetchall()]:
+        if not table_name.startswith(app_name):
+            continue    # skip tables not in this app
+        schema.append('table_name := '+ table_name)
+        cursor.execute("describe %s" % quote_name(table_name))
+        for row in cursor.fetchall():
+            tmp = []
+            for x in row:
+                tmp.append(str(x))
+            schema.append( '\t'.join(tmp) )
+        cursor.execute("SHOW INDEX FROM %s" % quote_name(table_name))
+        for row in cursor.fetchall():
+            schema.append( '\t'.join([ str(row[0]), str(row[1]), str(row[2]), str(row[3]), str(row[4]), str(row[5]), str(row[9]), ]) )
+        
+    return 'fv1:'+ str('\n'.join(schema).__hash__())
+
-    
-DATA_TYPES_REVERSE = {

django/branches/schema-evolution/docs/schema-evolution.txt

@@ -3 +3 @@
-== Introduction ==
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-== Downloading / Installing ==
@@ -25 +24 @@
-
-{{{
-
+branches/
-ln -s `pwd`/django_se_src/django SITE-PACKAGES-DIR/django
-}}}
@@ -51 +50 @@
-}}}
-
-
-
-is designed to be automagic via introspection.  M
-
-
-
+: Malcolm
+
+can be performed via introspection, as long as you're not doing anything too radical.  If you have an established application the ''vast'' majority of changes are either additions or renames (either tables or columns).  Or if you're new to SQL, introspection keeps things very simple for you.  To use schema evolution as Malcolm just m
+
+
+$ 
-}}}
-
@@ -87 +86 @@
-}}}
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-=== Adding / Removing Fields ===
@@ -368 +424 @@
-Correct, however I thought this to be a highly unlikely scenario, not warranting the extra notational complexity.  But just as we support strings and tuples, there is nothing to say we can't extend it to support say a mapping of historical names to date ranges, if the need arises.
-
+''The 'aka' approach is ambiguous for all but trivial use cases. It doesn't capture the idea that database changes occur in bulk, in sequence. For example, On Monday, I add two fields, remove 1 field, rename a table. That creates v2 of the database. On Tuesday, I bring back the deleted field, and remove one of the added fields, creating v3 of the database. This approach doesn't track which state a given database is in, and doesn't apply changes in blocks appropriate to versioned changes.''
+
+It does not matter how you get from v1 => v3, as long as you get there with minimum theoretical information loss.  The following:
+
+'''v1 => v2 => v3'''
+ 1. // v1 t1:{A}
+ 1. add_field(B);
+ 1. add_field(C);
+ 1. del_field(A);
+ 1. rename_table(t1,t2);
+ 1. // v2 t2{B,C}
+ 1. add_field(A);
+ 1. del_field(C);
+ 1. // v3 t2:{A,B}
+
+is functionally equivalent to:
+
+'''v1 => v3'''
+ 1. // v1 t1:{A}
+ 1. add_field(B);
+ 1. rename_table(t1,t2);
+ 1. // v3 t2:{A,B}
+
+And this can be supported completely through introspection + metadata about what tables and columns used to be called.  If you load v2 or v3, the available information can get you there from v1, and if you load v3, the available information can get you there from v1 or v2.
+
+A more detailed breakdown of this critique is available [http://kered.org/blog/2007-08-03/schema-evolution-confusion-example-case/ here], complete with working code examples.
+
-== Future Work ==
-
@@ -384 +467 @@
-Schema evolution should generate SQL to add the new column, push the data from the old to the new column, then delete the old column.  Warnings should be provided for completely incompatible types or other loss-of-information scenarios.
-
+The second biggest missing piece is foreign/m2m key support.
+
+Lastly, for the migration scripts, sometimes it's easier to write python than it is to write sql.  I intend for you to be able to interleave function calls in with the sql statements and have the schema evolution code just Do The Right Thing(tm).  But this isn't coded yet.
+
-== Conclusion ==
-
-
-
-
+
+