Django

Code

Changeset 5919

Show
Ignore:
Timestamp:
08/17/07 12:23:15 (11 months ago)
Author:
mtredinnick
Message:

Rewrote portions of the app- and model-cache initialisation to handle some corner cases. It is now possible to use m2m relations before everything is imported and still get the right results later when importing is complete. Also, get_apps() should always return the same results, so apps won't randomly disappear in the admin interface.

Also reorganised the structure of loading.py, since the number of global variables was exploding. The public API is still backwards compatible.

Fixed #1796 and #2438 (he claims, optimistically).

Files:

Legend:

Unmodified
Added
Removed
Modified
Copied
Moved

  • django/trunk/django/db/models/loading.py

    r5091 r5919  
    55import sys 
    66import os 
     7import threading 
    78 
    8 __all__ = ('get_apps', 'get_app', 'get_models', 'get_model', 'register_models') 
     9__all__ = ('get_apps', 'get_app', 'get_models', 'get_model', 'register_models', 
     10        'load_app', 'cache_ready') 
    911 
    10 _app_list = []   # Cache of installed apps. 
    11                  # Entry is not placed in app_list cache until entire app is loaded. 
    12 _app_models = {} # Dictionary of models against app label 
    13                  # Each value is a dictionary of model name: model class 
    14                  # Applabel and Model entry exists in cache when individual model is loaded. 
    15 _app_errors = {} # Dictionary of errors that were experienced when loading the INSTALLED_APPS 
    16                  # Key is the app_name of the model, value is the exception that was raised 
    17                  # during model loading. 
    18 _loaded = False  # Has the contents of settings.INSTALLED_APPS been loaded? 
    19                  # i.e., has get_apps() been called? 
     12class Cache(object): 
     13    """ 
     14    A cache that stores installed applications and their models. Used to 
     15    provide reverse-relations and for app introspection (e.g. admin). 
     16    """ 
     17    # Use the Borg pattern to share state between all instances. Details at 
     18    # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/66531. 
     19    __shared_state = dict( 
     20        # Keys of app_store are the model modules for each application. 
     21        app_store = {}, 
    2022 
    21 def get_apps(): 
    22     "Returns a list of all installed modules that contain models." 
    23     global _app_list 
    24     global _loaded 
    25     if not _loaded: 
    26         _loaded = True 
    27         for app_name in settings.INSTALLED_APPS: 
    28             try: 
    29                 load_app(app_name) 
    30             except Exception, e: 
    31                 # Problem importing the app 
    32                 _app_errors[app_name] = e 
    33     return _app_list 
     23        # Mapping of app_labels to a dictionary of model names to model code. 
     24        app_models = {}, 
    3425 
    35 def get_app(app_label, emptyOK=False): 
    36     "Returns the module containing the models for the given app_label. If the app has no models in it and 'emptyOK' is True, returns None." 
    37     get_apps() # Run get_apps() to populate the _app_list cache. Slightly hackish. 
    38     for app_name in settings.INSTALLED_APPS: 
    39         if app_label == app_name.split('.')[-1]: 
    40             mod = load_app(app_name) 
    41             if mod is None: 
    42                 if emptyOK: 
    43                     return None 
    44             else: 
    45                 return mod 
    46     raise ImproperlyConfigured, "App with label %s could not be found" % app_label 
     26        # Mapping of app_labels to errors raised when trying to import the app. 
     27        app_errors = {}, 
    4728 
    48 def load_app(app_name): 
    49     "Loads the app with the provided fully qualified name, and returns the model module." 
    50     global _app_list 
    51     mod = __import__(app_name, {}, {}, ['models']) 
    52     if not hasattr(mod, 'models'): 
    53         return None 
    54     if mod.models not in _app_list: 
    55         _app_list.append(mod.models) 
    56     return mod.models 
     29        # -- Everything below here is only used when populating the cache -- 
     30        loaded = False, 
     31        handled = {}, 
     32        postponed = [], 
     33        nesting_level = 0, 
     34        write_lock = threading.RLock(), 
     35    ) 
    5736 
    58 def get_app_errors(): 
    59     "Returns the map of known problems with the INSTALLED_APPS" 
    60     global _app_errors 
    61     get_apps() # Run get_apps() to populate the _app_list cache. Slightly hackish. 
    62     return _app_errors 
     37    def __init__(self): 
     38        self.__dict__ = self.__shared_state 
    6339 
    64 def get_models(app_mod=None): 
    65     """ 
    66     Given a module containing models, returns a list of the models. Otherwise 
    67     returns a list of all installed models. 
    68     """ 
    69     app_list = get_apps() # Run get_apps() to populate the _app_list cache. Slightly hackish. 
    70     if app_mod: 
    71         return _app_models.get(app_mod.__name__.split('.')[-2], {}).values() 
    72     else: 
    73         model_list = [] 
    74         for app_mod in app_list: 
    75             model_list.extend(get_models(app_mod)) 
    76         return model_list 
     40    def _populate(self): 
     41        """ 
     42        Fill in all the cache information. This method is threadsafe, in the 
     43        sense that every caller will see the same state upon return, and if the 
     44        cache is already initialised, it does no work. 
     45        """ 
     46        if self.loaded: 
     47            return 
     48        self.write_lock.acquire() 
     49        try: 
     50            if self.loaded: 
     51                return 
     52            for app_name in settings.INSTALLED_APPS: 
     53                if app_name in self.handled: 
     54                    continue 
     55                try: 
     56                    self.load_app(app_name, True) 
     57                except Exception, e: 
     58                    # Problem importing the app 
     59                    self.app_errors[app_name] = e 
     60            if not self.nesting_level: 
     61                for app_name in self.postponed: 
     62                    self.load_app(app_name) 
     63                self.loaded = True 
     64        finally: 
     65            self.write_lock.release() 
    7766 
    78 def get_model(app_label, model_name, seed_cache=True): 
    79     """ 
    80     Returns the model matching the given app_label and case-insensitive 
    81     model_name. 
     67    def load_app(self, app_name, can_postpone=False): 
     68        """ 
     69        Loads the app with the provided fully qualified name, and returns the 
     70        model module. 
     71        """ 
     72        self.handled[app_name] = None 
     73        self.nesting_level += 1 
     74        mod = __import__(app_name, {}, {}, ['models']) 
     75        self.nesting_level -= 1 
     76        if not hasattr(mod, 'models'): 
     77            if can_postpone: 
     78                # Either the app has no models, or the package is still being 
     79                # imported by Python and the model module isn't available yet. 
     80                # We will check again once all the recursion has finished (in 
     81                # populate). 
     82                self.postponed.append(app_name) 
     83            return None 
     84        if mod.models not in self.app_store: 
     85            self.app_store[mod.models] = len(self.app_store) 
     86        return mod.models 
    8287 
    83     Returns None if no model is found. 
    84     """ 
    85     if seed_cache: 
    86         get_apps() 
    87     try: 
    88         model_dict = _app_models[app_label] 
    89     except KeyError: 
    90         return None 
     88    def cache_ready(self): 
     89        """ 
     90        Returns true if the model cache is fully populated. 
    9191 
    92     try: 
    93         return model_dict[model_name.lower()] 
    94     except KeyError: 
    95         return None 
     92        Useful for code that wants to cache the results of get_models() for 
     93        themselves once it is safe to do so. 
     94        """ 
     95        return self.loaded 
    9696 
    97 def register_models(app_label, *models): 
    98     """ 
    99     Register a set of models as belonging to an app. 
    100     """ 
    101     for model in models: 
    102         # Store as 'name: model' pair in a dictionary 
    103         # in the _app_models dictionary 
    104         model_name = model._meta.object_name.lower() 
    105         model_dict = _app_models.setdefault(app_label, {}) 
    106         if model_name in model_dict: 
    107             # The same model may be imported via different paths (e.g. 
    108             # appname.models and project.appname.models). We use the source 
    109             # filename as a means to detect identity. 
    110             fname1 = os.path.abspath(sys.modules[model.__module__].__file__) 
    111             fname2 = os.path.abspath(sys.modules[model_dict[model_name].__module__].__file__) 
    112             # Since the filename extension could be .py the first time and .pyc 
    113             # or .pyo the second time, ignore the extension when comparing. 
    114             if os.path.splitext(fname1)[0] == os.path.splitext(fname2)[0]: 
    115                 continue 
    116         model_dict[model_name] = model 
     97    def get_apps(self): 
     98        "Returns a list of all installed modules that contain models." 
     99        self._populate() 
     100 
     101        # Ensure the returned list is always in the same order (with new apps 
     102        # added at the end). This avoids unstable ordering on the admin app 
     103        # list page, for example. 
     104        apps = [(v, k) for k, v in self.app_store.items()] 
     105        apps.sort() 
     106        return [elt[1] for elt in apps] 
     107 
     108    def get_app(self, app_label, emptyOK=False): 
     109        """ 
     110        Returns the module containing the models for the given app_label. If 
     111        the app has no models in it and 'emptyOK' is True, returns None. 
     112        """ 
     113        self._populate() 
     114        self.write_lock.acquire() 
     115        try: 
     116            for app_name in settings.INSTALLED_APPS: 
     117                if app_label == app_name.split('.')[-1]: 
     118                    mod = self.load_app(app_name, False) 
     119                    if mod is None: 
     120                        if emptyOK: 
     121                            return None 
     122                    else: 
     123                        return mod 
     124            raise ImproperlyConfigured, "App with label %s could not be found" % app_label 
     125        finally: 
     126            self.write_lock.release() 
     127 
     128    def get_app_errors(self): 
     129        "Returns the map of known problems with the INSTALLED_APPS." 
     130        self._populate() 
     131        return self.app_errors 
     132 
     133    def get_models(self, app_mod=None): 
     134        """ 
     135        Given a module containing models, returns a list of the models. 
     136        Otherwise returns a list of all installed models. 
     137        """ 
     138        self._populate() 
     139        if app_mod: 
     140            return self.app_models.get(app_mod.__name__.split('.')[-2], {}).values() 
     141        else: 
     142            model_list = [] 
     143            for app_entry in self.app_models.itervalues(): 
     144                model_list.extend(app_entry.values()) 
     145            return model_list 
     146 
     147    def get_model(self, app_label, model_name, seed_cache=True): 
     148        """ 
     149        Returns the model matching the given app_label and case-insensitive 
     150        model_name. 
     151 
     152        Returns None if no model is found. 
     153        """ 
     154        if seed_cache: 
     155            self._populate() 
     156        return self.app_models.get(app_label, {}).get(model_name.lower()) 
     157 
     158    def register_models(self, app_label, *models): 
     159        """ 
     160        Register a set of models as belonging to an app. 
     161        """ 
     162        for model in models: 
     163            # Store as 'name: model' pair in a dictionary 
     164            # in the _app_models dictionary 
     165            model_name = model._meta.object_name.lower() 
     166            model_dict = self.app_models.setdefault(app_label, {}) 
     167            if model_name in model_dict: 
     168                # The same model may be imported via different paths (e.g. 
     169                # appname.models and project.appname.models). We use the source 
     170                # filename as a means to detect identity. 
     171                fname1 = os.path.abspath(sys.modules[model.__module__].__file__) 
     172                fname2 = os.path.abspath(sys.modules[model_dict[model_name].__module__].__file__) 
     173                # Since the filename extension could be .py the first time and 
     174                # .pyc or .pyo the second time, ignore the extension when 
     175                # comparing. 
     176                if os.path.splitext(fname1)[0] == os.path.splitext(fname2)[0]: 
     177                    continue 
     178            model_dict[model_name] = model 
     179 
     180cache = Cache() 
     181 
     182# These methods were always module level, so are kept that way for backwards 
     183# compatibility. 
     184get_apps = cache.get_apps 
     185get_app = cache.get_app 
     186get_app_errors = cache.get_app_errors 
     187get_models = cache.get_models 
     188get_model = cache.get_model 
     189register_models = cache.register_models 
     190load_app = cache.load_app 
     191cache_ready = cache.cache_ready 

  • django/trunk/django/db/models/options.py

    r5609 r5919  
    33from django.db.models.fields.related import ManyToManyRel 
    44from django.db.models.fields import AutoField, FieldDoesNotExist 
    5 from django.db.models.loading import get_models 
     5from django.db.models.loading import get_models, cache_ready 
    66from django.db.models.query import orderlist2sql 
    77from django.db.models import Manager 
    …… 
    180180                    if f.rel and self == f.rel.to._meta: 
    181181                        rel_objs.append(RelatedObject(f.rel.to, klass, f)) 
    182             self._all_related_many_to_many_objects = rel_objs 
     182            if cache_ready(): 
     183                self._all_related_many_to_many_objects = rel_objs 
    183184            return rel_objs 
    184185