Ticket #3615: allow_forward_refs_in_fixtures_v7.diff

File allow_forward_refs_in_fixtures_v7.diff, 25.4 KB (added by Jim Dalton, 10 years ago)
  • django/core/management/commands/loaddata.py

    diff --git a/django/core/management/commands/loaddata.py b/django/core/management/commands/loaddata.py
    index 34f3543..38b57ed 100644
    a b class Command(BaseCommand): 
    166166                                    (format, fixture_name, humanize(fixture_dir)))
    167167                            try:
    168168                                objects = serializers.deserialize(format, fixture, using=using)
    169                                 for obj in objects:
    170                                     objects_in_fixture += 1
    171                                     if router.allow_syncdb(using, obj.object.__class__):
    172                                         loaded_objects_in_fixture += 1
    173                                         models.add(obj.object.__class__)
    174                                         obj.save(using=using)
     169                               
     170                                with connection.constraint_checks_disabled():
     171                                    for obj in objects:
     172                                        objects_in_fixture += 1
     173                                        if router.allow_syncdb(using, obj.object.__class__):
     174                                            loaded_objects_in_fixture += 1
     175                                            models.add(obj.object.__class__)
     176                                            obj.save(using=using)
     177                                       
     178                                # Since we disabled constraint checks, we must manually check for
     179                                # any invalid keys that might have been added
     180                                table_names = [model._meta.db_table for model in models]
     181                                connection.check_constraints(table_names=table_names)
     182                                   
    175183                                loaded_object_count += loaded_objects_in_fixture
    176184                                fixture_object_count += objects_in_fixture
    177185                                label_found = True
  • django/db/backends/__init__.py

    diff --git a/django/db/backends/__init__.py b/django/db/backends/__init__.py
    index b64fb01..7114e96 100644
    a b try: 
    44except ImportError:
    55    import dummy_thread as thread
    66from threading import local
     7from contextlib import contextmanager
    78
    89from django.conf import settings
    910from django.db import DEFAULT_DB_ALIAS
    class BaseDatabaseWrapper(local): 
    238239        """
    239240        if self.savepoint_state:
    240241            self._savepoint_commit(sid)
     242   
     243    @contextmanager
     244    def constraint_checks_disabled(self):
     245        disabled = self._disable_constraint_checking()
     246        try:
     247            yield
     248        finally:
     249            if disabled:
     250                self._enable_constraint_checking()
     251       
     252   
     253    def _disable_constraint_checking(self):
     254        """
     255        Backends can implement as needed to temporarily disable foreign key constraint
     256        checking.
     257        """
     258        pass
     259
     260    def _enable_constraint_checking(self):
     261        """
     262        Backends can implement as needed to re-enable foreign key constraint checking.
     263        """
     264        pass
     265   
     266    def check_constraints(self, table_names=None):
     267        """
     268        Backends can override this method if they can apply constraint checking (e.g. via "SET CONSTRAINTS
     269        ALL IMMEDIATE"). Should raise an IntegrityError if any invalid foreign key references are encountered.
     270        """
     271        pass
    241272
    242273    def close(self):
    243274        if self.connection is not None:
    class BaseDatabaseIntrospection(object): 
    870901
    871902        return sequence_list
    872903
     904    def get_key_columns(self, cursor, table_name):
     905        """
     906        Backends can override this to return a list of (column_name, referenced_table_name,
     907        referenced_column_name) for all key columns in given table.
     908        """
     909        raise NotImplementedError
     910   
     911    def get_primary_key_column(self, cursor, table_name):
     912        """
     913        Backends can override this to return the column name of the primary key for the given table.
     914        """
     915        raise NotImplementedError
     916
    873917class BaseDatabaseClient(object):
    874918    """
    875919    This class encapsulates all backend-specific methods for opening a
  • django/db/backends/dummy/base.py

    diff --git a/django/db/backends/dummy/base.py b/django/db/backends/dummy/base.py
    index 7de48c8..746f26b 100644
    a b class DatabaseIntrospection(BaseDatabaseIntrospection): 
    3434    get_table_description = complain
    3535    get_relations = complain
    3636    get_indexes = complain
     37    get_key_columns = complain
    3738
    3839class DatabaseWrapper(BaseDatabaseWrapper):
    3940    operators = {}
  • django/db/backends/mysql/base.py

    diff --git a/django/db/backends/mysql/base.py b/django/db/backends/mysql/base.py
    index 6d02aa7..c8e9dce 100644
    a b if (version < (1,2,1) or (version[:3] == (1, 2, 1) and 
    2525from MySQLdb.converters import conversions
    2626from MySQLdb.constants import FIELD_TYPE, CLIENT
    2727
    28 from django.db import utils
     28from django.db import utils, IntegrityError
    2929from django.db.backends import *
    3030from django.db.backends.signals import connection_created
    3131from django.db.backends.mysql.client import DatabaseClient
    class DatabaseWrapper(BaseDatabaseWrapper): 
    349349                raise Exception('Unable to determine MySQL version from version string %r' % self.connection.get_server_info())
    350350            self.server_version = tuple([int(x) for x in m.groups()])
    351351        return self.server_version
     352
     353    def _disable_constraint_checking(self):
     354        """
     355        Disables foreign key checks, primarily for use in adding rows with forward references. Always returns True,
     356        to indicate constraint checks need to be re-enabled.
     357        """
     358        self.cursor().execute('SET foreign_key_checks=0')
     359        return True
     360
     361    def _enable_constraint_checking(self):
     362        """
     363        Re-enable foreign key checks after they have been disabled.
     364        """
     365        self.cursor().execute('SET foreign_key_checks=1')
     366   
     367    def check_constraints(self, table_names):
     368        """
     369        Checks each table name in table-names for rows with invalid foreign key references. This method is
     370        intended to be used in conjunction with `disable_constraint_checking()` and `enable_constraint_checking()`, to
     371        determine if rows with invalid references were entered while constraint checks were off.
     372
     373        Raises an IntegrityError on the first invalid foreign key reference encountered (if any) and provides
     374        detailed information about the invalid reference in the error message.
     375
     376        Backends can override this method if they can more directly apply constraint checking (e.g. via "SET CONSTRAINTS
     377        ALL IMMEDIATE")
     378        """
     379        cursor = self.cursor()
     380        for table_name in table_names:
     381            primary_key_column_name = self.introspection.get_primary_key_column(cursor, table_name)
     382            if not primary_key_column_name:
     383                continue
     384            key_columns = self.introspection.get_key_columns(cursor, table_name)
     385            for column_name, referenced_table_name, referenced_column_name in key_columns:
     386                cursor.execute("""
     387                    SELECT REFERRING.`%s`, REFERRING.`%s` FROM `%s` as REFERRING
     388                    LEFT JOIN `%s` as REFERRED
     389                    ON (REFERRING.`%s` = REFERRED.`%s`)
     390                    WHERE REFERRING.`%s` IS NOT NULL
     391                        AND REFERRED.`%s` IS NULL"""
     392                    % (primary_key_column_name, column_name, table_name, referenced_table_name,
     393                       column_name, referenced_column_name, column_name, referenced_column_name))
     394                for bad_row in cursor.fetchall():
     395                    raise IntegrityError("The row in table '%s' with primary key '%s' has an invalid \
     396foreign key: %s.%s contains a value '%s' that does not have a corresponding value in %s.%s."
     397                                         % (table_name, bad_row[0], table_name, column_name, bad_row[1],
     398                                            referenced_table_name, referenced_column_name))
  • django/db/backends/mysql/introspection.py

    diff --git a/django/db/backends/mysql/introspection.py b/django/db/backends/mysql/introspection.py
    index 9e1518b..4612221 100644
    a b class DatabaseIntrospection(BaseDatabaseIntrospection): 
    5151        representing all relationships to the given table. Indexes are 0-based.
    5252        """
    5353        my_field_dict = self._name_to_index(cursor, table_name)
    54         constraints = []
     54        constraints = self.get_key_columns(cursor, table_name)
    5555        relations = {}
     56        for my_fieldname, other_table, other_field in constraints:
     57            other_field_index = self._name_to_index(cursor, other_table)[other_field]
     58            my_field_index = my_field_dict[my_fieldname]
     59            relations[my_field_index] = (other_field_index, other_table)
     60        return relations
     61
     62    def get_key_columns(self, cursor, table_name):
     63        """
     64        Returns a list of (column_name, referenced_table_name, referenced_column_name) for all
     65        key columns in given table.
     66        """
     67        key_columns = []
    5668        try:
    57             # This should work for MySQL 5.0.
    5869            cursor.execute("""
    5970                SELECT column_name, referenced_table_name, referenced_column_name
    6071                FROM information_schema.key_column_usage
    class DatabaseIntrospection(BaseDatabaseIntrospection): 
    6273                    AND table_schema = DATABASE()
    6374                    AND referenced_table_name IS NOT NULL
    6475                    AND referenced_column_name IS NOT NULL""", [table_name])
    65             constraints.extend(cursor.fetchall())
     76            key_columns.extend(cursor.fetchall())
    6677        except (ProgrammingError, OperationalError):
    6778            # Fall back to "SHOW CREATE TABLE", for previous MySQL versions.
    6879            # Go through all constraints and save the equal matches.
    class DatabaseIntrospection(BaseDatabaseIntrospection): 
    7485                    if match == None:
    7586                        break
    7687                    pos = match.end()
    77                     constraints.append(match.groups())
    78 
    79         for my_fieldname, other_table, other_field in constraints:
    80             other_field_index = self._name_to_index(cursor, other_table)[other_field]
    81             my_field_index = my_field_dict[my_fieldname]
    82             relations[my_field_index] = (other_field_index, other_table)
    83 
    84         return relations
     88                    key_columns.append(match.groups())
     89        return key_columns
     90   
     91    def get_primary_key_column(self, cursor, table_name):
     92        """
     93        Returns the name of the primary key column for the given table
     94        """
     95        for column in self.get_indexes(cursor, table_name).iteritems():
     96            if column[1]['primary_key']:
     97                return column[0]
     98        return None
    8599
    86100    def get_indexes(self, cursor, table_name):
    87101        """
  • django/db/backends/oracle/base.py

    diff --git a/django/db/backends/oracle/base.py b/django/db/backends/oracle/base.py
    index 42de0dd..c98be00 100644
    a b class DatabaseWrapper(BaseDatabaseWrapper): 
    428428        self.introspection = DatabaseIntrospection(self)
    429429        self.validation = BaseDatabaseValidation(self)
    430430
     431    def check_constraints(self, table_names=None):
     432        """
     433        To check constraints, we set constraints to immediate. Then, when, we're done we must ensure they
     434        are returned to deferred.
     435        """
     436        try:
     437            self.cursor().execute('SET CONSTRAINTS ALL IMMEDIATE')
     438        finally:
     439            self.cursor().execute('SET CONSTRAINTS ALL DEFERRED')
     440
    431441    def _valid_connection(self):
    432442        return self.connection is not None
    433443
  • django/db/backends/postgresql_psycopg2/base.py

    diff --git a/django/db/backends/postgresql_psycopg2/base.py b/django/db/backends/postgresql_psycopg2/base.py
    index d25a2fe..c970b8c 100644
    a b class DatabaseWrapper(BaseDatabaseWrapper): 
    106106        self.introspection = DatabaseIntrospection(self)
    107107        self.validation = BaseDatabaseValidation(self)
    108108        self._pg_version = None
     109   
     110    def check_constraints(self, table_names=None):
     111        """
     112        To check constraints, we set constraints to immediate. Then, when, we're done we must ensure they
     113        are returned to deferred.
     114        """
     115        try:
     116            self.cursor().execute('SET CONSTRAINTS ALL IMMEDIATE')
     117        finally:
     118            self.cursor().execute('SET CONSTRAINTS ALL DEFERRED')
    109119
    110120    def _get_pg_version(self):
    111121        if self._pg_version is None:
  • django/db/backends/sqlite3/base.py

    diff --git a/django/db/backends/sqlite3/base.py b/django/db/backends/sqlite3/base.py
    index 5b4a1c2..2a88fbd 100644
    a b class DatabaseWrapper(BaseDatabaseWrapper): 
    205205            connection_created.send(sender=self.__class__, connection=self)
    206206        return self.connection.cursor(factory=SQLiteCursorWrapper)
    207207
     208    def check_constraints(self, table_names):
     209        """
     210        Checks each table name in table-names for rows with invalid foreign key references. This method is
     211        intended to be used in conjunction with `disable_constraint_checking()` and `enable_constraint_checking()`, to
     212        determine if rows with invalid references were entered while constraint checks were off.
     213
     214        Raises an IntegrityError on the first invalid foreign key reference encountered (if any) and provides
     215        detailed information about the invalid reference in the error message.
     216
     217        Backends can override this method if they can more directly apply constraint checking (e.g. via "SET CONSTRAINTS
     218        ALL IMMEDIATE")
     219        """
     220        cursor = self.cursor()
     221        for table_name in table_names:
     222            primary_key_column_name = self.introspection.get_primary_key_column(cursor, table_name)
     223            if not primary_key_column_name:
     224                continue
     225            key_columns = self.introspection.get_key_columns(cursor, table_name)
     226            for column_name, referenced_table_name, referenced_column_name in key_columns:
     227                cursor.execute("""
     228                    SELECT REFERRING.`%s`, REFERRING.`%s` FROM `%s` as REFERRING
     229                    LEFT JOIN `%s` as REFERRED
     230                    ON (REFERRING.`%s` = REFERRED.`%s`)
     231                    WHERE REFERRING.`%s` IS NOT NULL
     232                        AND REFERRED.`%s` IS NULL"""
     233                    % (primary_key_column_name, column_name, table_name, referenced_table_name,
     234                       column_name, referenced_column_name, column_name, referenced_column_name))
     235                for bad_row in cursor.fetchall():
     236                    raise IntegrityError("The row in table '%s' with primary key '%s' has an invalid \
     237foreign key: %s.%s contains a value '%s' that does not have a corresponding value in %s.%s."
     238                                         % (table_name, bad_row[0], table_name, column_name, bad_row[1],
     239                                            referenced_table_name, referenced_column_name))
     240
    208241    def close(self):
    209242        # If database is in memory, closing the connection destroys the
    210243        # database. To prevent accidental data loss, ignore close requests on
  • django/db/backends/sqlite3/introspection.py

    diff --git a/django/db/backends/sqlite3/introspection.py b/django/db/backends/sqlite3/introspection.py
    index 5ee7b64..27de913 100644
    a b class DatabaseIntrospection(BaseDatabaseIntrospection): 
    103103
    104104        return relations
    105105
     106    def get_key_columns(self, cursor, table_name):
     107        """
     108        Returns a list of (column_name, referenced_table_name, referenced_column_name) for all
     109        key columns in given table.
     110        """
     111        key_columns = []
     112
     113        # Schema for this table
     114        cursor.execute("SELECT sql FROM sqlite_master WHERE tbl_name = %s AND type = %s", [table_name, "table"])
     115        results = cursor.fetchone()[0].strip()
     116        results = results[results.index('(')+1:results.rindex(')')]
     117
     118        # Walk through and look for references to other tables. SQLite doesn't
     119        # really have enforced references, but since it echoes out the SQL used
     120        # to create the table we can look for REFERENCES statements used there.
     121        for field_index, field_desc in enumerate(results.split(',')):
     122            field_desc = field_desc.strip()
     123            if field_desc.startswith("UNIQUE"):
     124                continue
     125
     126            m = re.search('"(.*)".*references (.*) \(["|](.*)["|]\)', field_desc, re.I)
     127            if not m:
     128                continue
     129
     130            # This will append (column_name, referenced_table_name, referenced_column_name) to key_columns
     131            key_columns.append(tuple([s.strip('"') for s in m.groups()]))
     132
     133        return key_columns
     134       
    106135    def get_indexes(self, cursor, table_name):
    107136        """
    108137        Returns a dictionary of fieldname -> infodict for the given table,
    class DatabaseIntrospection(BaseDatabaseIntrospection): 
    127156            name = info[0][2] # seqno, cid, name
    128157            indexes[name]['unique'] = True
    129158        return indexes
     159   
     160    def get_primary_key_column(self, cursor, table_name):
     161        """
     162        Get the column name of the primary key for the given table.
     163        """
     164        # Don't use PRAGMA because that causes issues with some transactions
     165        cursor.execute("SELECT sql FROM sqlite_master WHERE tbl_name = %s AND type = %s", [table_name, "table"])
     166        results = cursor.fetchone()[0].strip()
     167        results = results[results.index('(')+1:results.rindex(')')]
     168        for field_desc in results.split(','):
     169            field_desc = field_desc.strip()
     170            m = re.search('"(.*)".*PRIMARY KEY$', field_desc)
     171            if m:
     172                return m.groups()[0]
     173        return None
    130174
    131175    def _table_info(self, cursor, name):
    132176        cursor.execute('PRAGMA table_info(%s)' % self.connection.ops.quote_name(name))
  • docs/ref/databases.txt

    diff --git a/docs/ref/databases.txt b/docs/ref/databases.txt
    index 2f55b9c..5a2042a 100644
    a b currently the only engine that supports full-text indexing and searching. 
    142142The InnoDB_ engine is fully transactional and supports foreign key references
    143143and is probably the best choice at this point in time.
    144144
     145.. versionchanged:: 1.4
     146
     147In previous versions of Django, fixtures with forward references (i.e.
     148relations to rows that have not yet been inserted into the database) would fail
     149to load when using the InnoDB storage engine. This was due to the fact that InnoDB
     150deviates from the SQL standard by checking foreign key constraints immediately
     151instead of deferring the check until the transaction is committed. This
     152problem has been resolved in Django 1.4. Fixture data is now loaded with foreign key
     153checks turned off; foreign key checks are then re-enabled when the data has
     154finished loading, at which point the entire table is checked for invalid foreign
     155key references and an `IntegrityError` is raised if any are found.
     156
    145157.. _storage engines: http://dev.mysql.com/doc/refman/5.5/en/storage-engines.html
    146158.. _MyISAM: http://dev.mysql.com/doc/refman/5.5/en/myisam-storage-engine.html
    147159.. _InnoDB: http://dev.mysql.com/doc/refman/5.5/en/innodb.html
  • tests/modeltests/serializers/tests.py

    diff --git a/tests/modeltests/serializers/tests.py b/tests/modeltests/serializers/tests.py
    index 013438e..70f3bf9 100644
    a b from xml.dom import minidom 
    55
    66from django.conf import settings
    77from django.core import serializers
    8 from django.db import transaction
     8from django.db import transaction, connection
    99from django.test import TestCase, TransactionTestCase, Approximate
    1010from django.utils import simplejson, unittest
    1111
    class SerializersTransactionTestBase(object): 
    252252        transaction.enter_transaction_management()
    253253        transaction.managed(True)
    254254        objs = serializers.deserialize(self.serializer_name, self.fwd_ref_str)
    255         for obj in objs:
    256             obj.save()
     255        with connection.constraint_checks_disabled():
     256            for obj in objs:
     257                obj.save()
    257258        transaction.commit()
    258259        transaction.leave_transaction_management()
    259260
  • new file tests/regressiontests/fixtures_regress/fixtures/forward_ref.json

    diff --git a/tests/regressiontests/fixtures_regress/fixtures/forward_ref.json b/tests/regressiontests/fixtures_regress/fixtures/forward_ref.json
    new file mode 100644
    index 0000000..237b076
    - +  
     1[
     2    {
     3        "pk": 1,
     4        "model": "fixtures_regress.book",
     5        "fields": {
     6            "name": "Cryptonomicon",
     7            "author": 4
     8        }
     9    },
     10    {
     11        "pk": "4",
     12        "model": "fixtures_regress.person",
     13        "fields": {
     14            "name": "Neal Stephenson"
     15        }
     16    }
     17]
     18 No newline at end of file
  • new file tests/regressiontests/fixtures_regress/fixtures/forward_ref_bad_data.json

    diff --git a/tests/regressiontests/fixtures_regress/fixtures/forward_ref_bad_data.json b/tests/regressiontests/fixtures_regress/fixtures/forward_ref_bad_data.json
    new file mode 100644
    index 0000000..3a3fb64
    - +  
     1[
     2    {
     3        "pk": 1,
     4        "model": "fixtures_regress.book",
     5        "fields": {
     6            "name": "Cryptonomicon",
     7            "author": 3
     8        }
     9    },
     10    {
     11        "pk": "4",
     12        "model": "fixtures_regress.person",
     13        "fields": {
     14            "name": "Neal Stephenson"
     15        }
     16    }
     17]
     18 No newline at end of file
  • tests/regressiontests/fixtures_regress/tests.py

    diff --git a/tests/regressiontests/fixtures_regress/tests.py b/tests/regressiontests/fixtures_regress/tests.py
    index 3dc4ede..76d640e 100644
    a b from django.core import management 
    1212from django.core.management.commands.dumpdata import sort_dependencies
    1313from django.core.management.base import CommandError
    1414from django.db.models import signals
    15 from django.db import transaction
     15from django.db import transaction, IntegrityError
    1616from django.test import TestCase, TransactionTestCase, skipIfDBFeature, \
    1717    skipUnlessDBFeature
    1818
    class TestFixtures(TestCase): 
    362362            """[{"pk": %d, "model": "fixtures_regress.widget", "fields": {"name": "grommet"}}]"""
    363363            % widget.pk
    364364            )
     365   
     366    def test_loaddata_works_when_fixture_has_forward_refs(self):
     367        """
     368        Regression for #3615 - Forward references cause fixtures not to load in MySQL (InnoDB)
     369        """
     370        management.call_command(
     371            'loaddata',
     372            'forward_ref.json',
     373            verbosity=0,
     374            commit=False
     375        )
     376        self.assertEqual(Book.objects.all()[0].id, 1)
     377        self.assertEqual(Person.objects.all()[0].id, 4)
     378   
     379    def test_loaddata_raises_error_when_fixture_has_invalid_foreign_key(self):
     380        """
     381        Regression for #3615 - Ensure data with nonexistent child key references raises error
     382        """
     383        stderr = StringIO()
     384        management.call_command(
     385            'loaddata',
     386            'forward_ref_bad_data.json',
     387            verbosity=0,
     388            commit=False,
     389            stderr=stderr,
     390        )
     391        self.assertTrue(
     392            stderr.getvalue().startswith('Problem installing fixture')
     393        )
    365394
    366395
    367396class NaturalKeyFixtureTests(TestCase):
  • tests/regressiontests/introspection/tests.py

    diff --git a/tests/regressiontests/introspection/tests.py b/tests/regressiontests/introspection/tests.py
    index e309a98..09c6eda 100644
    a b class IntrospectionTests(TestCase): 
    9696            # That's {field_index: (field_index_other_table, other_table)}
    9797            self.assertEqual(relations, {3: (0, Reporter._meta.db_table)})
    9898
     99    def test_get_key_columns(self):
     100        cursor = connection.cursor()
     101        key_columns = connection.introspection.get_key_columns(cursor, Article._meta.db_table)
     102        self.assertEqual(key_columns, [(u'reporter_id', Reporter._meta.db_table, u'id')])
     103   
     104    def test_get_primary_key_column(self):
     105        cursor = connection.cursor()
     106        primary_key_column = connection.introspection.get_primary_key_column(cursor, Article._meta.db_table)
     107        self.assertEqual(primary_key_column, u'id')
     108
    99109    def test_get_indexes(self):
    100110        cursor = connection.cursor()
    101111        indexes = connection.introspection.get_indexes(cursor, Article._meta.db_table)
  • tests/regressiontests/serializers_regress/tests.py

    diff --git a/tests/regressiontests/serializers_regress/tests.py b/tests/regressiontests/serializers_regress/tests.py
    index 90a438c..9ec9510 100644
    a b def serializerTest(format, self): 
    383383    objects = []
    384384    instance_count = {}
    385385    for (func, pk, klass, datum) in test_data:
    386         objects.extend(func[0](pk, klass, datum))
     386        with connection.constraint_checks_disabled():
     387            objects.extend(func[0](pk, klass, datum))
    387388
    388389    # Get a count of the number of objects created for each class
    389390    for klass in instance_count:
Back to Top