Ticket #3615: allow_forward_refs_in_fixtures_v4.diff

File allow_forward_refs_in_fixtures_v4.diff, 16.6 KB (added by Jim Dalton, 10 years ago)
  • docs/ref/databases.txt

     
    142142The InnoDB_ engine is fully transactional and supports foreign key references
    143143and is probably the best choice at this point in time.
    144144
     145.. versionchanged:: 1.4
     146
     147In previous versions of Django, fixtures with forward references (i.e.
     148relations to rows that have not yet been inserted into the database) would fail
     149to load when using the InnoDB storage engine. This was due to the fact that InnoDB
     150deviates from the SQL standard by checking foreign key constraints immediately
     151instead of deferring the check until the transaction is committed. This
     152problem has been resolved in Django 1.4. Fixture data is now loaded with foreign key
     153checks turned off; foreign key checks are then re-enabled when the data has
     154finished loading, at which point the entire table is checked for invalid foreign
     155key references and an `IntegrityError` is raised if any are found.
     156
    145157.. _storage engines: http://dev.mysql.com/doc/refman/5.5/en/storage-engines.html
    146158.. _MyISAM: http://dev.mysql.com/doc/refman/5.5/en/myisam-storage-engine.html
    147159.. _InnoDB: http://dev.mysql.com/doc/refman/5.5/en/innodb.html
  • django/db/backends/mysql/base.py

     
    2525from MySQLdb.converters import conversions
    2626from MySQLdb.constants import FIELD_TYPE, CLIENT
    2727
    28 from django.db import utils
     28from django.db import utils, IntegrityError
    2929from django.db.backends import *
    3030from django.db.backends.signals import connection_created
    3131from django.db.backends.mysql.client import DatabaseClient
     
    349349                raise Exception('Unable to determine MySQL version from version string %r' % self.connection.get_server_info())
    350350            self.server_version = tuple([int(x) for x in m.groups()])
    351351        return self.server_version
     352
     353    def disable_foreign_key_checks(self):
     354        """
     355        Disable foreign key checks, primarily for us in adding rows with forward references. Always returns True,
     356        to indicate checks have been disabled.
     357        """
     358        self.foreign_key_checks_disabled = True
     359        self.cursor().execute('SET foreign_key_checks=0')
     360        return True
     361
     362    def enable_foreign_key_checks(self):
     363        """
     364        Re-enable foreign key checks after they have been disabled. Always returns True,
     365        to indicate checks have been re-enabled.
     366        """
     367        self.foreign_key_checks_disabled = False
     368        self.cursor().execute('SET foreign_key_checks=1')
     369        return True
     370   
     371    def check_for_invalid_foreign_keys(self, table_name):
     372        """
     373        Checks given table for rows with invalid foreign key references. This method is intended to be used in
     374        conjunction with `disable_foreign_keys()` and `enable_foreign_keys()`, to determine if rows with
     375        invalid references were entered while foreign key checks were off.
     376       
     377        Raises an IntegrityError on the first invalid foreign key reference encountered (if any) and provides
     378        detailed information about the invalid reference in the error message.
     379        """
     380        cursor = self.cursor()
     381        key_columns = self.introspection.get_key_columns(cursor, table_name)
     382        for column_name, referenced_table_name, referenced_column_name in key_columns:
     383            primary_key_column_name = [c[0] for c in self.introspection.get_indexes(cursor, table_name).iteritems() if c[1]['primary_key']][0]
     384            cursor.execute("""
     385                SELECT REFERRING.`%s`, REFERRING.`%s` FROM `%s` as REFERRING
     386                LEFT JOIN `%s` as REFERRED
     387                ON (REFERRING.`%s` = REFERRED.`%s`)
     388                WHERE REFERRING.`%s` IS NOT NULL
     389                    AND REFERRED.`%s` IS NULL"""
     390                % (primary_key_column_name, column_name, table_name, referenced_table_name,
     391                   column_name, referenced_column_name, column_name, referenced_column_name))
     392            for bad_row in cursor.fetchall():
     393                raise IntegrityError("The row in table '%s' with primary key '%s' has an invalid \
     394foreign key: %s.%s contains a value '%s' that does not have a corresponding value in %s.%s."
     395                                     % (table_name, bad_row[0], table_name, column_name, bad_row[1],
     396                                        referenced_table_name, referenced_column_name))
  • django/db/backends/mysql/introspection.py

     
    5151        representing all relationships to the given table. Indexes are 0-based.
    5252        """
    5353        my_field_dict = self._name_to_index(cursor, table_name)
    54         constraints = []
     54        constraints = self.get_key_columns(cursor, table_name)
    5555        relations = {}
     56        for my_fieldname, other_table, other_field in constraints:
     57            other_field_index = self._name_to_index(cursor, other_table)[other_field]
     58            my_field_index = my_field_dict[my_fieldname]
     59            relations[my_field_index] = (other_field_index, other_table)
     60        return relations
     61
     62    def get_key_columns(self, cursor, table_name):
     63        """
     64        Returns a list of [column_name, referenced_table_name, referenced_column_name] for all
     65        key columns in given table.
     66        """
     67        key_columns = []
    5668        try:
    57             # This should work for MySQL 5.0.
    5869            cursor.execute("""
    5970                SELECT column_name, referenced_table_name, referenced_column_name
    6071                FROM information_schema.key_column_usage
     
    6273                    AND table_schema = DATABASE()
    6374                    AND referenced_table_name IS NOT NULL
    6475                    AND referenced_column_name IS NOT NULL""", [table_name])
    65             constraints.extend(cursor.fetchall())
     76            key_columns.extend(cursor.fetchall())
    6677        except (ProgrammingError, OperationalError):
    6778            # Fall back to "SHOW CREATE TABLE", for previous MySQL versions.
    6879            # Go through all constraints and save the equal matches.
     
    7485                    if match == None:
    7586                        break
    7687                    pos = match.end()
    77                     constraints.append(match.groups())
     88                    key_columns.append(match.groups())
     89        return key_columns
    7890
    79         for my_fieldname, other_table, other_field in constraints:
    80             other_field_index = self._name_to_index(cursor, other_table)[other_field]
    81             my_field_index = my_field_dict[my_fieldname]
    82             relations[my_field_index] = (other_field_index, other_table)
    83 
    84         return relations
    85 
    8691    def get_indexes(self, cursor, table_name):
    8792        """
    8893        Returns a dictionary of fieldname -> infodict for the given table,
  • django/db/backends/__init__.py

     
    3434        self.transaction_state = []
    3535        self.savepoint_state = 0
    3636        self._dirty = None
     37       
     38        self.foreign_key_checks_disabled = False
    3739
    3840    def __eq__(self, other):
    3941        return self.alias == other.alias
     
    239241        if self.savepoint_state:
    240242            self._savepoint_commit(sid)
    241243
     244    def disable_foreign_key_checks(self):
     245        """
     246        Backends can implement as needed to temporarily disable foreign key constraint
     247        checking.
     248        """
     249        pass
     250
     251    def enable_foreign_key_checks(self):
     252        """
     253        Backends can implement as needed to re-enable foreign key constraint
     254        checking.
     255        """
     256        pass
     257   
     258    def check_for_invalid_foreign_keys(self, table_name):
     259        """
     260        Backends can implement as needed to raise IntegrityError if invalid foreign keys
     261        are found in a table (for use when adding rows with foreign key checks disabled)
     262        """
     263        pass
     264
    242265    def close(self):
    243266        if self.connection is not None:
    244267            self.connection.close()
     
    870893
    871894        return sequence_list
    872895
     896    def get_key_columns(self, cursor, table_name):
     897        raise NotImplementedError
     898
    873899class BaseDatabaseClient(object):
    874900    """
    875901    This class encapsulates all backend-specific methods for opening a
  • django/db/backends/dummy/base.py

     
    3434    get_table_description = complain
    3535    get_relations = complain
    3636    get_indexes = complain
     37    get_key_columns = complain
    3738
    3839class DatabaseWrapper(BaseDatabaseWrapper):
    3940    operators = {}
  • django/core/management/commands/loaddata.py

     
    88from django.core import serializers
    99from django.core.management.base import BaseCommand
    1010from django.core.management.color import no_style
    11 from django.db import connections, router, transaction, DEFAULT_DB_ALIAS
     11from django.db import connections, router, transaction, DEFAULT_DB_ALIAS, models as db_models
    1212from django.db.models import get_apps
    1313from django.utils.itercompat import product
    1414
     
    166166                                    (format, fixture_name, humanize(fixture_dir)))
    167167                            try:
    168168                                objects = serializers.deserialize(format, fixture, using=using)
     169                                foreign_key_checks_disabled = connection.disable_foreign_key_checks()
    169170                                for obj in objects:
    170171                                    objects_in_fixture += 1
    171172                                    if router.allow_syncdb(using, obj.object.__class__):
    172173                                        loaded_objects_in_fixture += 1
    173174                                        models.add(obj.object.__class__)
    174175                                        obj.save(using=using)
     176                                       
     177                                # If we disabled foreign key checks, then we should re-enable them and check for
     178                                # any invalid keys that might have been added
     179                                if foreign_key_checks_disabled:
     180                                    connection.enable_foreign_key_checks()
     181                                    for model in models:
     182                                        connection.check_for_invalid_foreign_keys(table_name=model._meta.db_table)
     183                                   
    175184                                loaded_object_count += loaded_objects_in_fixture
    176185                                fixture_object_count += objects_in_fixture
    177186                                label_found = True
  • tests/modeltests/serializers/tests.py

     
    55
    66from django.conf import settings
    77from django.core import serializers
    8 from django.db import transaction
     8from django.db import transaction, connection
    99from django.test import TestCase, TransactionTestCase, Approximate
    1010from django.utils import simplejson, unittest
    1111
     
    252252        transaction.enter_transaction_management()
    253253        transaction.managed(True)
    254254        objs = serializers.deserialize(self.serializer_name, self.fwd_ref_str)
     255        connection.disable_foreign_key_checks()
    255256        for obj in objs:
    256257            obj.save()
     258        connection.enable_foreign_key_checks()
    257259        transaction.commit()
    258260        transaction.leave_transaction_management()
    259261
  • tests/regressiontests/serializers_regress/tests.py

     
    383383    objects = []
    384384    instance_count = {}
    385385    for (func, pk, klass, datum) in test_data:
     386        connection.disable_foreign_key_checks()
    386387        objects.extend(func[0](pk, klass, datum))
     388        connection.enable_foreign_key_checks()
    387389
    388390    # Get a count of the number of objects created for each class
    389391    for klass in instance_count:
  • tests/regressiontests/introspection/tests.py

     
    9696            # That's {field_index: (field_index_other_table, other_table)}
    9797            self.assertEqual(relations, {3: (0, Reporter._meta.db_table)})
    9898
     99    def test_get_key_columns(self):
     100        cursor = connection.cursor()
     101        key_columns = connection.introspection.get_key_columns(cursor, Article._meta.db_table)
     102        self.assertEqual(key_columns, [(u'reporter_id', Reporter._meta.db_table, u'id')])
     103
    99104    def test_get_indexes(self):
    100105        cursor = connection.cursor()
    101106        indexes = connection.introspection.get_indexes(cursor, Article._meta.db_table)
  • tests/regressiontests/fixtures_regress/fixtures/forward_ref_bad_data.json

     
     1[
     2    {
     3        "pk": 1,
     4        "model": "fixtures_regress.book",
     5        "fields": {
     6            "name": "Cryptonomicon",
     7            "author": 3
     8        }
     9    },
     10    {
     11        "pk": "4",
     12        "model": "fixtures_regress.person",
     13        "fields": {
     14            "name": "Neal Stephenson"
     15        }
     16    }
     17]
     18 No newline at end of file
  • tests/regressiontests/fixtures_regress/fixtures/forward_ref.json

     
     1[
     2    {
     3        "pk": 1,
     4        "model": "fixtures_regress.book",
     5        "fields": {
     6            "name": "Cryptonomicon",
     7            "author": 4
     8        }
     9    },
     10    {
     11        "pk": "4",
     12        "model": "fixtures_regress.person",
     13        "fields": {
     14            "name": "Neal Stephenson"
     15        }
     16    }
     17]
     18 No newline at end of file
  • tests/regressiontests/fixtures_regress/tests.py

     
    1212from django.core.management.commands.dumpdata import sort_dependencies
    1313from django.core.management.base import CommandError
    1414from django.db.models import signals
    15 from django.db import transaction
     15from django.db import transaction, IntegrityError
    1616from django.test import TestCase, TransactionTestCase, skipIfDBFeature, \
    1717    skipUnlessDBFeature
    1818
     
    362362            """[{"pk": %d, "model": "fixtures_regress.widget", "fields": {"name": "grommet"}}]"""
    363363            % widget.pk
    364364            )
     365   
     366    def test_loaddata_works_when_fixture_has_forward_refs(self):
     367        """
     368        Regression for #3615 - Forward references cause fixtures not to load in MySQL (InnoDB)
     369        """
     370        management.call_command(
     371            'loaddata',
     372            'forward_ref.json',
     373            verbosity=0,
     374            commit=False
     375        )
     376        self.assertEqual(Book.objects.all()[0].id, 1)
     377        self.assertEqual(Person.objects.all()[0].id, 4)
     378   
     379    def test_loaddata_raise_when_fixture_has_forward_refs(self):
     380        """
     381        Regression for #3615 - Ensure data with nonexistent child key references raises error
     382        """
     383        stderr = StringIO()
     384        management.call_command(
     385            'loaddata',
     386            'forward_ref_bad_data.json',
     387            verbosity=0,
     388            commit=False,
     389            stderr=stderr,
     390        )
     391        self.assertTrue(
     392            stderr.getvalue().startswith('Problem installing fixture')
     393        )
    365394
    366395
    367396class NaturalKeyFixtureTests(TestCase):
Back to Top