Ticket #3615: allow_forward_refs_in_fixtures_v5.diff

File allow_forward_refs_in_fixtures_v5.diff, 19.0 KB (added by Jim Dalton, 10 years ago)
  • docs/ref/databases.txt

     
    142142The InnoDB_ engine is fully transactional and supports foreign key references
    143143and is probably the best choice at this point in time.
    144144
     145.. versionchanged:: 1.4
     146
     147In previous versions of Django, fixtures with forward references (i.e.
     148relations to rows that have not yet been inserted into the database) would fail
     149to load when using the InnoDB storage engine. This was due to the fact that InnoDB
     150deviates from the SQL standard by checking foreign key constraints immediately
     151instead of deferring the check until the transaction is committed. This
     152problem has been resolved in Django 1.4. Fixture data is now loaded with foreign key
     153checks turned off; foreign key checks are then re-enabled when the data has
     154finished loading, at which point the entire table is checked for invalid foreign
     155key references and an `IntegrityError` is raised if any are found.
     156
    145157.. _storage engines: http://dev.mysql.com/doc/refman/5.5/en/storage-engines.html
    146158.. _MyISAM: http://dev.mysql.com/doc/refman/5.5/en/myisam-storage-engine.html
    147159.. _InnoDB: http://dev.mysql.com/doc/refman/5.5/en/innodb.html
  • django/db/backends/sqlite3/base.py

     
    204204            self.connection.create_function("django_format_dtdelta", 5, _sqlite_format_dtdelta)
    205205            connection_created.send(sender=self.__class__, connection=self)
    206206        return self.connection.cursor(factory=SQLiteCursorWrapper)
    207 
     207   
    208208    def close(self):
    209209        # If database is in memory, closing the connection destroys the
    210210        # database. To prevent accidental data loss, ignore close requests on
  • django/db/backends/sqlite3/introspection.py

     
    103103
    104104        return relations
    105105
     106    def get_key_columns(self, cursor, table_name):
     107        """
     108        Returns a list of (column_name, referenced_table_name, referenced_column_name) for all
     109        key columns in given table.
     110        """
     111        key_columns = []
     112
     113        # Schema for this table
     114        cursor.execute("SELECT sql FROM sqlite_master WHERE tbl_name = %s AND type = %s", [table_name, "table"])
     115        results = cursor.fetchone()[0].strip()
     116        results = results[results.index('(')+1:results.rindex(')')]
     117
     118        # Walk through and look for references to other tables. SQLite doesn't
     119        # really have enforced references, but since it echoes out the SQL used
     120        # to create the table we can look for REFERENCES statements used there.
     121        for field_index, field_desc in enumerate(results.split(',')):
     122            field_desc = field_desc.strip()
     123            if field_desc.startswith("UNIQUE"):
     124                continue
     125
     126            m = re.search('"(.*)".*references (.*) \(["|](.*)["|]\)', field_desc, re.I)
     127            if not m:
     128                continue
     129
     130            # This will append (column_name, referenced_table_name, referenced_column_name) to key_columns
     131            key_columns.append(tuple([s.strip('"') for s in m.groups()]))
     132
     133        return key_columns
     134       
    106135    def get_indexes(self, cursor, table_name):
    107136        """
    108137        Returns a dictionary of fieldname -> infodict for the given table,
  • django/db/backends/mysql/base.py

     
    2525from MySQLdb.converters import conversions
    2626from MySQLdb.constants import FIELD_TYPE, CLIENT
    2727
    28 from django.db import utils
     28from django.db import utils, IntegrityError
    2929from django.db.backends import *
    3030from django.db.backends.signals import connection_created
    3131from django.db.backends.mysql.client import DatabaseClient
     
    349349                raise Exception('Unable to determine MySQL version from version string %r' % self.connection.get_server_info())
    350350            self.server_version = tuple([int(x) for x in m.groups()])
    351351        return self.server_version
     352
     353    def disable_constraint_checking(self):
     354        """
     355        Disables foreign key checks, primarily for use in adding rows with forward references. Always returns True,
     356        to indicate constraint checks have been disabled.
     357        """
     358        self.cursor().execute('SET foreign_key_checks=0')
     359        return True
     360
     361    def enable_constraint_checking(self):
     362        """
     363        Re-enable foreign key checks after they have been disabled. Always returns True,
     364        to indicate checks have been re-enabled.
     365        """
     366        self.cursor().execute('SET foreign_key_checks=1')
     367        return True
  • django/db/backends/mysql/introspection.py

     
    5151        representing all relationships to the given table. Indexes are 0-based.
    5252        """
    5353        my_field_dict = self._name_to_index(cursor, table_name)
    54         constraints = []
     54        constraints = self.get_key_columns(cursor, table_name)
    5555        relations = {}
     56        for my_fieldname, other_table, other_field in constraints:
     57            other_field_index = self._name_to_index(cursor, other_table)[other_field]
     58            my_field_index = my_field_dict[my_fieldname]
     59            relations[my_field_index] = (other_field_index, other_table)
     60        return relations
     61
     62    def get_key_columns(self, cursor, table_name):
     63        """
     64        Returns a list of (column_name, referenced_table_name, referenced_column_name) for all
     65        key columns in given table.
     66        """
     67        key_columns = []
    5668        try:
    57             # This should work for MySQL 5.0.
    5869            cursor.execute("""
    5970                SELECT column_name, referenced_table_name, referenced_column_name
    6071                FROM information_schema.key_column_usage
     
    6273                    AND table_schema = DATABASE()
    6374                    AND referenced_table_name IS NOT NULL
    6475                    AND referenced_column_name IS NOT NULL""", [table_name])
    65             constraints.extend(cursor.fetchall())
     76            key_columns.extend(cursor.fetchall())
    6677        except (ProgrammingError, OperationalError):
    6778            # Fall back to "SHOW CREATE TABLE", for previous MySQL versions.
    6879            # Go through all constraints and save the equal matches.
     
    7485                    if match == None:
    7586                        break
    7687                    pos = match.end()
    77                     constraints.append(match.groups())
     88                    key_columns.append(match.groups())
     89        return key_columns
    7890
    79         for my_fieldname, other_table, other_field in constraints:
    80             other_field_index = self._name_to_index(cursor, other_table)[other_field]
    81             my_field_index = my_field_dict[my_fieldname]
    82             relations[my_field_index] = (other_field_index, other_table)
    83 
    84         return relations
    85 
    8691    def get_indexes(self, cursor, table_name):
    8792        """
    8893        Returns a dictionary of fieldname -> infodict for the given table,
  • django/db/backends/__init__.py

     
    239239        if self.savepoint_state:
    240240            self._savepoint_commit(sid)
    241241
     242    def disable_constraint_checking(self):
     243        """
     244        Backends can implement as needed to temporarily disable foreign key constraint
     245        checking. If a backend does not require post-load constraint checking, then this method
     246        can be overridden and False can be returned.
     247        """
     248        return True
     249
     250    def enable_constraint_checking(self):
     251        """
     252        Backends can implement as needed to re-enable foreign key constraint checking.
     253        """
     254        pass
     255   
     256    def check_constraints(self, table_names):
     257        """
     258        Checks each table name in table-names for rows with invalid foreign key references. This method is
     259        intended to be used in conjunction with `disable_constraint_checking()` and `enable_constraint_checking()`, to
     260        determine if rows with invalid references were entered while constraint checks were off.
     261
     262        Raises an IntegrityError on the first invalid foreign key reference encountered (if any) and provides
     263        detailed information about the invalid reference in the error message.
     264       
     265        Backends can override this method if they can more directly apply constraint checking (e.g. via "SET CONSTRAINTS
     266        ALL IMMEDIATE")
     267        """
     268        cursor = self.cursor()
     269        for table_name in table_names:
     270            key_columns = self.introspection.get_key_columns(cursor, table_name)
     271            for column_name, referenced_table_name, referenced_column_name in key_columns:
     272                primary_key_column_name = [c[0] for c in self.introspection.get_indexes(cursor, table_name).iteritems() if c[1]['primary_key']][0]
     273                cursor.execute("""
     274                    SELECT REFERRING.`%s`, REFERRING.`%s` FROM `%s` as REFERRING
     275                    LEFT JOIN `%s` as REFERRED
     276                    ON (REFERRING.`%s` = REFERRED.`%s`)
     277                    WHERE REFERRING.`%s` IS NOT NULL
     278                        AND REFERRED.`%s` IS NULL"""
     279                    % (primary_key_column_name, column_name, table_name, referenced_table_name,
     280                       column_name, referenced_column_name, column_name, referenced_column_name))
     281                for bad_row in cursor.fetchall():
     282                    raise IntegrityError("The row in table '%s' with primary key '%s' has an invalid \
     283foreign key: %s.%s contains a value '%s' that does not have a corresponding value in %s.%s."
     284                                         % (table_name, bad_row[0], table_name, column_name, bad_row[1],
     285                                            referenced_table_name, referenced_column_name))
     286
    242287    def close(self):
    243288        if self.connection is not None:
    244289            self.connection.close()
     
    870915
    871916        return sequence_list
    872917
     918    def get_key_columns(self, cursor, table_name):
     919        """
     920        Backends should override this to return a list of (column_name, referenced_table_name,
     921        referenced_column_name) for all key columns in given table.
     922        """
     923        return []
     924
    873925class BaseDatabaseClient(object):
    874926    """
    875927    This class encapsulates all backend-specific methods for opening a
  • django/db/backends/dummy/base.py

     
    3434    get_table_description = complain
    3535    get_relations = complain
    3636    get_indexes = complain
     37    get_key_columns = complain
    3738
    3839class DatabaseWrapper(BaseDatabaseWrapper):
    3940    operators = {}
  • django/core/management/commands/loaddata.py

     
    88from django.core import serializers
    99from django.core.management.base import BaseCommand
    1010from django.core.management.color import no_style
    11 from django.db import connections, router, transaction, DEFAULT_DB_ALIAS
     11from django.db import connections, router, transaction, DEFAULT_DB_ALIAS, models as db_models
    1212from django.db.models import get_apps
    1313from django.utils.itercompat import product
    1414
     
    166166                                    (format, fixture_name, humanize(fixture_dir)))
    167167                            try:
    168168                                objects = serializers.deserialize(format, fixture, using=using)
     169                                constraint_checking_disabled = connection.disable_constraint_checking()
    169170                                for obj in objects:
    170171                                    objects_in_fixture += 1
    171172                                    if router.allow_syncdb(using, obj.object.__class__):
    172173                                        loaded_objects_in_fixture += 1
    173174                                        models.add(obj.object.__class__)
    174175                                        obj.save(using=using)
     176                                       
     177                                # If we disabled constraint checks, then we should re-enable them and check for
     178                                # any invalid keys that might have been added
     179                                if constraint_checking_disabled:
     180                                    connection.enable_constraint_checking()
     181                                    table_names = [model._meta.db_table for model in models]
     182                                    connection.check_constraints(table_names=table_names)
     183                                   
    175184                                loaded_object_count += loaded_objects_in_fixture
    176185                                fixture_object_count += objects_in_fixture
    177186                                label_found = True
  • tests/modeltests/serializers/tests.py

     
    55
    66from django.conf import settings
    77from django.core import serializers
    8 from django.db import transaction
     8from django.db import transaction, connection
    99from django.test import TestCase, TransactionTestCase, Approximate
    1010from django.utils import simplejson, unittest
    1111
     
    252252        transaction.enter_transaction_management()
    253253        transaction.managed(True)
    254254        objs = serializers.deserialize(self.serializer_name, self.fwd_ref_str)
     255        connection.disable_constraint_checking()
    255256        for obj in objs:
    256257            obj.save()
     258        connection.enable_constraint_checking()
    257259        transaction.commit()
    258260        transaction.leave_transaction_management()
    259261
  • tests/regressiontests/serializers_regress/tests.py

     
    383383    objects = []
    384384    instance_count = {}
    385385    for (func, pk, klass, datum) in test_data:
     386        connection.disable_constraint_checking()
    386387        objects.extend(func[0](pk, klass, datum))
     388        connection.enable_constraint_checking()
    387389
    388390    # Get a count of the number of objects created for each class
    389391    for klass in instance_count:
  • tests/regressiontests/introspection/tests.py

     
    9696            # That's {field_index: (field_index_other_table, other_table)}
    9797            self.assertEqual(relations, {3: (0, Reporter._meta.db_table)})
    9898
     99    def test_get_key_columns(self):
     100        cursor = connection.cursor()
     101        key_columns = connection.introspection.get_key_columns(cursor, Article._meta.db_table)
     102        self.assertEqual(key_columns, [(u'reporter_id', Reporter._meta.db_table, u'id')])
     103
    99104    def test_get_indexes(self):
    100105        cursor = connection.cursor()
    101106        indexes = connection.introspection.get_indexes(cursor, Article._meta.db_table)
  • tests/regressiontests/fixtures_regress/fixtures/forward_ref_bad_data.json

     
     1[
     2    {
     3        "pk": 1,
     4        "model": "fixtures_regress.book",
     5        "fields": {
     6            "name": "Cryptonomicon",
     7            "author": 3
     8        }
     9    },
     10    {
     11        "pk": "4",
     12        "model": "fixtures_regress.person",
     13        "fields": {
     14            "name": "Neal Stephenson"
     15        }
     16    }
     17]
     18 No newline at end of file
  • tests/regressiontests/fixtures_regress/fixtures/forward_ref.json

     
     1[
     2    {
     3        "pk": 1,
     4        "model": "fixtures_regress.book",
     5        "fields": {
     6            "name": "Cryptonomicon",
     7            "author": 4
     8        }
     9    },
     10    {
     11        "pk": "4",
     12        "model": "fixtures_regress.person",
     13        "fields": {
     14            "name": "Neal Stephenson"
     15        }
     16    }
     17]
     18 No newline at end of file
  • tests/regressiontests/fixtures_regress/tests.py

     
    1212from django.core.management.commands.dumpdata import sort_dependencies
    1313from django.core.management.base import CommandError
    1414from django.db.models import signals
    15 from django.db import transaction
     15from django.db import transaction, IntegrityError
    1616from django.test import TestCase, TransactionTestCase, skipIfDBFeature, \
    1717    skipUnlessDBFeature
    1818
     
    362362            """[{"pk": %d, "model": "fixtures_regress.widget", "fields": {"name": "grommet"}}]"""
    363363            % widget.pk
    364364            )
     365   
     366    def test_loaddata_works_when_fixture_has_forward_refs(self):
     367        """
     368        Regression for #3615 - Forward references cause fixtures not to load in MySQL (InnoDB)
     369        """
     370        management.call_command(
     371            'loaddata',
     372            'forward_ref.json',
     373            verbosity=0,
     374            commit=False
     375        )
     376        self.assertEqual(Book.objects.all()[0].id, 1)
     377        self.assertEqual(Person.objects.all()[0].id, 4)
     378   
     379    def test_loaddata_raises_error_when_fixture_has_invalid_foreign_key(self):
     380        """
     381        Regression for #3615 - Ensure data with nonexistent child key references raises error
     382        """
     383        stderr = StringIO()
     384        management.call_command(
     385            'loaddata',
     386            'forward_ref_bad_data.json',
     387            verbosity=0,
     388            commit=False,
     389            stderr=stderr,
     390        )
     391        self.assertTrue(
     392            stderr.getvalue().startswith('Problem installing fixture')
     393        )
    365394
    366395
    367396class NaturalKeyFixtureTests(TestCase):
Back to Top