Ticket #20485: 20485-prewalk-fixtures-dirs.diff

django/core/management/commands/loaddata.py

@@ -12 +12 @@
 from django.core.management.color import no_style
 from django.db import (connections, router, transaction, DEFAULT_DB_ALIAS,
       IntegrityError, DatabaseError)
-from django.db.models import get_apps
+from django.db.models import get_app_paths
 from django.utils.encoding import force_text
+from django.utils.functional import cached_property
 from django.utils._os import upath
+from django.utils import six
 from itertools import product
 
 try:
@@ -69 +71 @@
         self.fixture_object_count = 0
         self.models = set()
 
-        class SingleZipReader(zipfile.ZipFile):
-            def __init__(self, *args, **kwargs):
-                zipfile.ZipFile.__init__(self, *args, **kwargs)
-                if settings.DEBUG:
-                    assert len(self.namelist()) == 1, "Zip-compressed fixtures must contain only one file."
-            def read(self):
-                return zipfile.ZipFile.read(self, self.namelist()[0])
-
         self.compression_types = {
             None:   open,
             'gz':   gzip.GzipFile,
@@ -85 +79 @@
         if has_bz2:
             self.compression_types['bz2'] = bz2.BZ2File
 
-        app_module_paths = []
-        for app in get_apps():
-            if hasattr(app, '__path__'):
-                # It's a 'models/' subpackage
-                for path in app.__path__:
-                    app_module_paths.append(upath(path))
-            else:
-                # It's a models.py module
-                app_module_paths.append(upath(app.__file__))
-
-        app_fixtures = [os.path.join(os.path.dirname(path), 'fixtures') for path in app_module_paths]
-
         with connection.constraint_checks_disabled():
             for fixture_label in fixture_labels:
-                self.load_label(fixture_label, app_fixtures)
+                self.load_label(fixture_label)
 
         # Since we disabled constraint checks, we must manually check for
         # any invalid keys that might have been added
@@ -130 +112 @@
                 self.stdout.write("Installed %d object(s) (of %d) from %d fixture(s)" % (
                     self.loaded_object_count, self.fixture_object_count, self.fixture_count))
 
-    def load_label(self, fixture_label, app_fixtures):
+    def load_label(self, fixture_label):
 
         parts = fixture_label.split('.')
 
@@ -159 +141 @@
                     (fixture_name, format))
 
         if os.path.isabs(fixture_name):
-            fixture_dirs = [fixture_name]
+            dirs_candidates = {'': [fixture_name]}
         else:
-            fixture_dirs = app_fixtures + list(settings.FIXTURE_DIRS) + ['']
+            dirs_candidates = self.fixture_files_cache
 
         label_found = False
-        for fixture_dir in fixture_dirs:
-            found = self.process_dir(fixture_dir, fixture_name,
+        for fixture_dir, candidates in six.iteritems(dirs_candidates):
+            found = self.process_dir(fixture_dir, candidates, fixture_name,
                 compression_formats, formats)
             label_found = label_found or found
 
         if fixture_name != 'initial_data' and not label_found:
             warnings.warn("No fixture named '%s' found." % fixture_name)
 
-    def process_dir(self, fixture_dir, fixture_name, compression_formats,
-                    serialization_formats):
+    def process_dir(self, fixture_dir, candidates, fixture_name,
+                compression_formats, serialization_formats):
 
         humanize = lambda dirname: "'%s'" % dirname if dirname else 'absolute path'
 
@@ -193 +175 @@
             if self.verbosity >= 3:
                 self.stdout.write("Trying %s for %s fixture '%s'..." % \
                     (humanize(fixture_dir), file_name, fixture_name))
+            if file_name not in candidates:
+                continue
             full_path = os.path.join(fixture_dir, file_name)
             open_method = self.compression_types[compression_format]
             try:
@@ -250 +234 @@
                             (fixture_name))
 
         return label_found
+
+    @cached_property
+    def fixture_files_cache(self):
+        """
+        Return a dict mapping fixture directories to a list of their files.
+
+        Fixtures directories includes the 'fixtures' subdirectories of
+        installed application and the directories in FIXTURE_DIRS.
+
+        File paths are relative to the fixture directory containing them.
+        """
+        dirs = []
+        for path in get_app_paths():
+            d = os.path.join(os.path.dirname(path), 'fixtures')
+            if os.path.isdir(d):
+                dirs.append(d)
+        dirs.extend(list(settings.FIXTURE_DIRS))
+        dirs = [os.path.abspath(os.path.realpath(d)) for d in dirs]
+
+        cache = {}
+        for d in dirs:
+            files = []
+            for dirpath, dirnames, filenames in os.walk(d):
+                dirpath = os.path.relpath(dirpath, d)
+                if dirpath == '.':
+                    dirpath = ''
+                for filename in filenames:
+                    files.append(os.path.join(dirpath, filename))
+            cache[d] = files
+
+        return cache
+
+
+class SingleZipReader(zipfile.ZipFile):
+
+    def __init__(self, *args, **kwargs):
+        zipfile.ZipFile.__init__(self, *args, **kwargs)
+        if len(self.namelist()) != 1:
+            raise ValueError("Zip-compressed fixtures must contain one file.")
+
+    def read(self):
+        return zipfile.ZipFile.read(self, self.namelist()[0])

django/db/models/__init__.py

@@ -1 +1 @@
 from functools import wraps
 
 from django.core.exceptions import ObjectDoesNotExist, ImproperlyConfigured
-from django.db.models.loading import get_apps, get_app, get_models, get_model, register_models
+from django.db.models.loading import get_apps, get_app_paths, get_app, get_models, get_model, register_models
 from django.db.models.query import Q
 from django.db.models.expressions import F
 from django.db.models.manager import Manager

django/db/models/loading.py

@@ -130 +130 @@
         return self.loaded
 
     def get_apps(self):
-        "Returns a list of all installed modules that contain models."
+        """
+        Returns a list of all installed modules that contain models.
+        """
         self._populate()
 
         # Ensure the returned list is always in the same order (with new apps
@@ -140 +142 @@
         apps.sort()
         return [elt[1] for elt in apps]
 
+    def get_app_paths(self):
+        """
+        Returns a list of paths to all installed apps.
+
+        Useful for discovering files at conventional locations inside apps
+        (static files, templates, etc.)
+        """
+        self._populate()
+
+        app_paths = []
+        for app in self.get_apps():
+            if hasattr(app, '__path__'):        # models/__init__.py package
+                app_paths.extend([upath(path) for path in app.__path__])
+            else:                               # models.py module
+                app_paths.append(upath(app.__file__))
+        return app_paths
+
     def get_app(self, app_label, emptyOK=False):
         """
         Returns the module containing the models for the given app_label. If
@@ -260 +279 @@
 # These methods were always module level, so are kept that way for backwards
 # compatibility.
 get_apps = cache.get_apps
+get_app_paths = cache.get_app_paths
 get_app = cache.get_app
 get_app_errors = cache.get_app_errors
 get_models = cache.get_models