Ticket #14442: t14442.diff

File t14442.diff, 19.2 KB (added by Alex Gaynor, 14 years ago)

Patch implements the outlines strategy, 25% gain on admin_views, negligible of the full test run. 6 failing tests.

  • django/core/management/commands/loaddata.py 

    diff --git a/django/core/management/commands/loaddata.py b/django/core/management/commands/loaddata.py
index 34f3543..4451a77 100644
    a b  
    1 import sys
    21import os
    32import gzip
    43import zipfile
    … … from optparse import make_option  
    65
    76from django.conf import settings
    87from django.core import serializers
    9 from django.core.management.base import BaseCommand
     8from django.core.management.base import BaseCommand, CommandError
    109from django.core.management.color import no_style
    1110from django.db import connections, router, transaction, DEFAULT_DB_ALIAS
    1211from django.db.models import get_apps
    … … try:  
    1817except ImportError:
    1918    has_bz2 = False
    2019
     20
     21class SingleZipReader(zipfile.ZipFile):
     22    def __init__(self, *args, **kwargs):
     23        zipfile.ZipFile.__init__(self, *args, **kwargs)
     24        if settings.DEBUG:
     25            assert len(self.namelist()) == 1, "Zip-compressed fixtures must contain only one file."
     26
     27    def read(self):
     28        return zipfile.ZipFile.read(self, self.namelist()[0])
     29
     30def humanize(dirname):
     31    if dirname:
     32        return "'%s'" % dirname
     33    return 'absolute path'
     34
     35def find_fixture_data(fixture_labels, verbosity, using, stdout):
     36    app_module_paths = []
     37    for app in get_apps():
     38        if hasattr(app, '__path__'):
     39            # It's a 'models/' subpackage
     40            app_module_paths.extend(app.__path__)
     41        else:
     42            # It's a models.py module
     43            app_module_paths.append(app.__file__)
     44    app_fixtures = [
     45        os.path.join(os.path.dirname(path), 'fixtures')
     46        for path in app_module_paths
     47    ]
     48
     49    compression_types = {
     50        None:   file,
     51        'gz':   gzip.GzipFile,
     52        'zip':  SingleZipReader
     53    }
     54    if has_bz2:
     55        compression_types['bz2'] = bz2.BZ2File
     56
     57    objs = []
     58    models = set()
     59    fixture_count = 0
     60    found_object_count = 0
     61    fixture_object_count = 0
     62
     63    for fixture_label in fixture_labels:
     64        parts = fixture_label.split('.')
     65
     66        if len(parts) > 1 and parts[-1] in compression_types:
     67            compression_formats = [parts[-1]]
     68            parts = parts[:-1]
     69        else:
     70            compression_formats = compression_types.keys()
     71
     72        if len(parts) == 1:
     73            fixture_name = parts[0]
     74            formats = serializers.get_public_serializer_formats()
     75        else:
     76            fixture_name, format = '.'.join(parts[:-1]), parts[-1]
     77            if format in serializers.get_public_serializer_formats():
     78                formats = [format]
     79            else:
     80                formats = []
     81
     82        if formats:
     83            if verbosity >= 2:
     84                stdout.write("Loading '%s' fixtures...\n" % fixture_name)
     85        else:
     86            raise CommandError("Problem installing fixture '%s': %s is not a "
     87                "known serialization format." % (fixture_name, format))
     88
     89        if os.path.isabs(fixture_name):
     90            fixture_dirs = [fixture_name]
     91        else:
     92            fixture_dirs = app_fixtures + list(settings.FIXTURE_DIRS) + ['']
     93
     94        for fixture_dir in fixture_dirs:
     95            if verbosity >= 2:
     96                stdout.write("Checking %s for fixtures...\n" % humanize(fixture_dir))
     97
     98            label_found = False
     99            for combo in product([using, None], formats, compression_formats):
     100                database, format, compression_format = combo
     101                file_name = '.'.join(
     102                    p for p in [
     103                        fixture_name, database, format, compression_format
     104                    ]
     105                    if p
     106                )
     107
     108                if verbosity >= 3:
     109                    stdout.write("Trying %s for %s fixture '%s'...\n" % \
     110                        (humanize(fixture_dir), file_name, fixture_name))
     111                full_path = os.path.join(fixture_dir, file_name)
     112                open_method = compression_types[compression_format]
     113                try:
     114                    fixture = open_method(full_path, 'r')
     115                except IOError:
     116                    if verbosity >= 2:
     117                        stdout.write("No %s fixture '%s' in %s.\n" % \
     118                            (format, fixture_name, humanize(fixture_dir)))
     119                    continue
     120
     121                if label_found:
     122                    fixture.close()
     123                    raise CommandError("Multiple fixtures named '%s' in %s."
     124                        " Aborting" % (fixture_name, humanize(fixture_dir)))
     125                fixture_count += 1
     126                objects_in_fixture = 0
     127                found_objects_in_fixture = 0
     128                if verbosity >= 2:
     129                    stdout.write("Installing %s fixture '%s' from %s.\n" % \
     130                        (format, fixture_name, humanize(fixture_dir)))
     131                objects = serializers.deserialize(format, fixture, using=using)
     132                for obj in objects:
     133                    objects_in_fixture += 1
     134                    if router.allow_syncdb(using, obj.object.__class__):
     135                        found_objects_in_fixture += 1
     136                        models.add(obj.object.__class__)
     137                        objs.append(obj)
     138                found_object_count += found_objects_in_fixture
     139                fixture_object_count += objects_in_fixture
     140                label_found = True
     141                fixture.close()
     142
     143                # If the fixture we loaded contains 0 objects, assume that an
     144                # error was encountered during fixture loading.
     145                if objects_in_fixture == 0:
     146                    raise CommandError("No fixture data found for '%s'. "
     147                        "(File format may be invalid.)" % fixture_name)
     148    return objs, models, fixture_count, fixture_object_count
     149
    21150class Command(BaseCommand):
    22151    help = 'Installs the named fixture(s) in the database.'
    23152    args = "fixture [fixture ...]"
    … … class Command(BaseCommand):  
    45174        # the transaction in place when loaddata was invoked.
    46175        commit = options.get('commit', True)
    47176
    48         # Keep a count of the installed objects and fixtures
    49         fixture_count = 0
    50         loaded_object_count = 0
    51         fixture_object_count = 0
    52         models = set()
    53 
    54         humanize = lambda dirname: dirname and "'%s'" % dirname or 'absolute path'
    55 
    56177        # Get a cursor (even though we don't need one yet). This has
    57178        # the side effect of initializing the test database (if
    58179        # it isn't already initialized).
    … … class Command(BaseCommand):  
    65186            transaction.enter_transaction_management(using=using)
    66187            transaction.managed(True, using=using)
    67188
    68         class SingleZipReader(zipfile.ZipFile):
    69             def __init__(self, *args, **kwargs):
    70                 zipfile.ZipFile.__init__(self, *args, **kwargs)
    71                 if settings.DEBUG:
    72                     assert len(self.namelist()) == 1, "Zip-compressed fixtures must contain only one file."
    73             def read(self):
    74                 return zipfile.ZipFile.read(self, self.namelist()[0])
    75 
    76         compression_types = {
    77             None:   file,
    78             'gz':   gzip.GzipFile,
    79             'zip':  SingleZipReader
    80         }
    81         if has_bz2:
    82             compression_types['bz2'] = bz2.BZ2File
    83 
    84         app_module_paths = []
    85         for app in get_apps():
    86             if hasattr(app, '__path__'):
    87                 # It's a 'models/' subpackage
    88                 for path in app.__path__:
    89                     app_module_paths.append(path)
    90             else:
    91                 # It's a models.py module
    92                 app_module_paths.append(app.__file__)
    93 
    94         app_fixtures = [os.path.join(os.path.dirname(path), 'fixtures') for path in app_module_paths]
    95         for fixture_label in fixture_labels:
    96             parts = fixture_label.split('.')
    97 
    98             if len(parts) > 1 and parts[-1] in compression_types:
    99                 compression_formats = [parts[-1]]
    100                 parts = parts[:-1]
    101             else:
    102                 compression_formats = compression_types.keys()
    103 
    104             if len(parts) == 1:
    105                 fixture_name = parts[0]
    106                 formats = serializers.get_public_serializer_formats()
    107             else:
    108                 fixture_name, format = '.'.join(parts[:-1]), parts[-1]
    109                 if format in serializers.get_public_serializer_formats():
    110                     formats = [format]
    111                 else:
    112                     formats = []
    113 
    114             if formats:
    115                 if verbosity >= 2:
    116                     self.stdout.write("Loading '%s' fixtures...\n" % fixture_name)
    117             else:
    118                 self.stderr.write(
    119                     self.style.ERROR("Problem installing fixture '%s': %s is not a known serialization format.\n" %
    120                         (fixture_name, format)))
    121                 if commit:
    122                     transaction.rollback(using=using)
    123                     transaction.leave_transaction_management(using=using)
    124                 return
    125 
    126             if os.path.isabs(fixture_name):
    127                 fixture_dirs = [fixture_name]
    128             else:
    129                 fixture_dirs = app_fixtures + list(settings.FIXTURE_DIRS) + ['']
    130 
    131             for fixture_dir in fixture_dirs:
    132                 if verbosity >= 2:
    133                     self.stdout.write("Checking %s for fixtures...\n" % humanize(fixture_dir))
    134 
    135                 label_found = False
    136                 for combo in product([using, None], formats, compression_formats):
    137                     database, format, compression_format = combo
    138                     file_name = '.'.join(
    139                         p for p in [
    140                             fixture_name, database, format, compression_format
    141                         ]
    142                         if p
    143                     )
    144 
    145                     if verbosity >= 3:
    146                         self.stdout.write("Trying %s for %s fixture '%s'...\n" % \
    147                             (humanize(fixture_dir), file_name, fixture_name))
    148                     full_path = os.path.join(fixture_dir, file_name)
    149                     open_method = compression_types[compression_format]
    150                     try:
    151                         fixture = open_method(full_path, 'r')
    152                         if label_found:
    153                             fixture.close()
    154                             self.stderr.write(self.style.ERROR("Multiple fixtures named '%s' in %s. Aborting.\n" %
    155                                 (fixture_name, humanize(fixture_dir))))
    156                             if commit:
    157                                 transaction.rollback(using=using)
    158                                 transaction.leave_transaction_management(using=using)
    159                             return
    160                         else:
    161                             fixture_count += 1
    162                             objects_in_fixture = 0
    163                             loaded_objects_in_fixture = 0
    164                             if verbosity >= 2:
    165                                 self.stdout.write("Installing %s fixture '%s' from %s.\n" % \
    166                                     (format, fixture_name, humanize(fixture_dir)))
    167                             try:
    168                                 objects = serializers.deserialize(format, fixture, using=using)
    169                                 for obj in objects:
    170                                     objects_in_fixture += 1
    171                                     if router.allow_syncdb(using, obj.object.__class__):
    172                                         loaded_objects_in_fixture += 1
    173                                         models.add(obj.object.__class__)
    174                                         obj.save(using=using)
    175                                 loaded_object_count += loaded_objects_in_fixture
    176                                 fixture_object_count += objects_in_fixture
    177                                 label_found = True
    178                             except (SystemExit, KeyboardInterrupt):
    179                                 raise
    180                             except Exception:
    181                                 import traceback
    182                                 fixture.close()
    183                                 if commit:
    184                                     transaction.rollback(using=using)
    185                                     transaction.leave_transaction_management(using=using)
    186                                 if show_traceback:
    187                                     traceback.print_exc()
    188                                 else:
    189                                     self.stderr.write(
    190                                         self.style.ERROR("Problem installing fixture '%s': %s\n" %
    191                                              (full_path, ''.join(traceback.format_exception(sys.exc_type,
    192                                                  sys.exc_value, sys.exc_traceback)))))
    193                                 return
    194                             fixture.close()
    195 
    196                             # If the fixture we loaded contains 0 objects, assume that an
    197                             # error was encountered during fixture loading.
    198                             if objects_in_fixture == 0:
    199                                 self.stderr.write(
    200                                     self.style.ERROR("No fixture data found for '%s'. (File format may be invalid.)\n" %
    201                                         (fixture_name)))
    202                                 if commit:
    203                                     transaction.rollback(using=using)
    204                                     transaction.leave_transaction_management(using=using)
    205                                 return
    206 
    207                     except Exception, e:
    208                         if verbosity >= 2:
    209                             self.stdout.write("No %s fixture '%s' in %s.\n" % \
    210                                 (format, fixture_name, humanize(fixture_dir)))
     189        try:
     190            objs, models, fixture_count, fixture_object_count = find_fixture_data(
     191                fixture_labels,
     192                verbosity=verbosity,
     193                using=using,
     194                stdout=self.stdout
     195            )
     196        except CommandError, e:
     197            if commit:
     198                transaction.rollback(using=using)
     199                transaction.leave_transaction_management(using=using)
     200            # For reasons I don't understand CommandErrors are never raised.
     201            self.stderr.write(e.args[0] + "\n")
     202            return
     203        for obj in objs:
     204            obj.save(using=using)
    211205
    212206        # If we found even one object in a fixture, we need to reset the
    213207        # database sequences.
    214         if loaded_object_count > 0:
     208        if objs:
    215209            sequence_sql = connection.ops.sequence_reset_sql(self.style, models)
    216210            if sequence_sql:
    217211                if verbosity >= 2:
    … … class Command(BaseCommand):  
    228222                self.stdout.write("No fixtures found.\n")
    229223        else:
    230224            if verbosity >= 1:
    231                 if fixture_object_count == loaded_object_count:
     225                if fixture_object_count == len(objs):
    232226                    self.stdout.write("Installed %d object(s) from %d fixture(s)\n" % (
    233                         loaded_object_count, fixture_count))
     227                        len(objs), fixture_count))
    234228                else:
    235229                    self.stdout.write("Installed %d object(s) (of %d) from %d fixture(s)\n" % (
    236                         loaded_object_count, fixture_object_count, fixture_count))
     230                        len(objs), fixture_object_count, fixture_count))
    237231
    238232        # Close the DB connection. This is required as a workaround for an
    239233        # edge case in MySQL: if the same connection is used to

  • django/test/testcases.py 

    diff --git a/django/test/testcases.py b/django/test/testcases.py
index 65664a1..3cd5a31 100644
    a b from xml.dom.minidom import parseString, Node  
    66from django.conf import settings
    77from django.core import mail
    88from django.core.management import call_command
     9from django.core.management.commands.loaddata import find_fixture_data
    910from django.core.urlresolvers import clear_url_caches
    1011from django.db import transaction, connection, connections, DEFAULT_DB_ALIAS
    1112from django.http import QueryDict
    … … class TransactionTestCase(unittest.TestCase):  
    238239    # Can be overridden in derived classes.
    239240    client_class = Client
    240241
     242    @classmethod
     243    def setUpClass(cls):
     244        cls._fixture_objs = {}
     245        if hasattr(cls, "fixtures"):
     246            for db in cls._get_dbs():
     247                cls._fixture_objs[db] = find_fixture_data(
     248                    cls.fixtures, verbosity=0, using=db, stdout=sys.stdout
     249                )[0]
     250
     251    @classmethod
     252    def tearDownClass(cls):
     253        # To save memory I suppose.
     254        del cls._fixture_objs
     255
     256    @classmethod
     257    def _get_dbs(cls):
     258        if getattr(cls, 'multi_db', False):
     259            return connections
     260        return [DEFAULT_DB_ALIAS]
     261
     262    def _load_fixture_objs(self, db):
     263        for obj in self._fixture_objs.get(db, []):
     264            # DeserializedObject.save clears m2m_data, we get it back in there
     265            m2m_data = obj.m2m_data
     266            obj.save(using=db)
     267            obj.m2m_data = m2m_data
     268
    241269    def _pre_setup(self):
    242270        """Performs any pre-test setup. This includes:
    243271
    … … class TransactionTestCase(unittest.TestCase):  
    255283    def _fixture_setup(self):
    256284        # If the test case has a multi_db=True flag, flush all databases.
    257285        # Otherwise, just flush default.
    258         if getattr(self, 'multi_db', False):
    259             databases = connections
    260         else:
    261             databases = [DEFAULT_DB_ALIAS]
    262         for db in databases:
     286        for db in self._get_dbs():
    263287            call_command('flush', verbosity=0, interactive=False, database=db)
    264 
    265             if hasattr(self, 'fixtures'):
    266                 # We have to use this slightly awkward syntax due to the fact
    267                 # that we're using *args and **kwargs together.
    268                 call_command('loaddata', *self.fixtures, **{'verbosity': 0, 'database': db})
     288            self._load_fixture_objs(db)
    269289
    270290    def _urlconf_setup(self):
    271291        if hasattr(self, 'urls'):
    … … class TestCase(TransactionTestCase):  
    535555
    536556        # If the test case has a multi_db=True flag, setup all databases.
    537557        # Otherwise, just use default.
    538         if getattr(self, 'multi_db', False):
    539             databases = connections
    540         else:
    541             databases = [DEFAULT_DB_ALIAS]
    542 
     558        databases = self._get_dbs()
    543559        for db in databases:
    544560            transaction.enter_transaction_management(using=db)
    545561            transaction.managed(True, using=db)
    … … class TestCase(TransactionTestCase):  
    549565        Site.objects.clear_cache()
    550566
    551567        for db in databases:
    552             if hasattr(self, 'fixtures'):
    553                 call_command('loaddata', *self.fixtures, **{
    554                                                             'verbosity': 0,
    555                                                             'commit': False,
    556                                                             'database': db
    557                                                             })
     568            self._load_fixture_objs(db)
    558569
    559570    def _fixture_teardown(self):
    560571        if not connections_support_transactions():
Back to Top