Ticket #1186: query-rework.patch

File query-rework.patch, 21.8 KB (added by freakboy@…, 10 years ago)

Patch for primary key lookup in parse_lookup

  • django/db/models/options.py

     
    166166        return follow
    167167
    168168    def get_all_related_many_to_many_objects(self):
    169         module_list = get_installed_model_modules()
    170         rel_objs = []
    171         for mod in module_list:
    172             for klass in mod._MODELS:
    173                 for f in klass._meta.many_to_many:
    174                     if f.rel and self == f.rel.to._meta:
    175                         rel_objs.append(RelatedObject(self, klass, f))
    176         return rel_objs
     169        try: # Try the cache first.
     170            return self._all_related_many_to_many_objects
     171        except AttributeError:
     172            module_list = get_installed_model_modules()
     173            rel_objs = []
     174            for mod in module_list:
     175                for klass in mod._MODELS:
     176                    for f in klass._meta.many_to_many:
     177                        if f.rel and self == f.rel.to._meta:
     178                            rel_objs.append(RelatedObject(self, klass, f))
     179            self._all_related_many_to_many_objects = rel_objs
     180            return rel_objs
    177181
    178182    def get_ordered_objects(self):
    179183        "Returns a list of Options objects that are ordered with respect to this object."
  • django/db/models/query.py

     
    170170            select.extend(['%s.%s' % (backend.quote_name(db_table), backend.quote_name(f2.column)) for f2 in f.rel.to._meta.fields])
    171171            fill_table_cache(f.rel.to._meta, select, tables, where, db_table, cache_tables_seen)
    172172
    173 def throw_bad_kwarg_error(kwarg):
    174     # Helper function to remove redundancy.
    175     raise TypeError, "got unexpected keyword argument '%s'" % kwarg
     173   
     174class SortedDict(dict):
     175    "A dictionary that can keep its keys in the order in which they are inserted."
     176    def __init__(self, data={}):
     177        dict.__init__(self, data)
     178        self.keyOrder = data.keys()
     179    def __setitem__(self, key, value):
     180        dict.__setitem__(self, key, value)
     181        if key not in self.keyOrder:
     182            self.keyOrder.append(key)
     183    def __delitem__(self, key, value):
     184        dict.__delitem__(self, key, value)
     185        self.keyOrder.remove(key)
     186    def __iter__(self):
     187        for k in self.keyOrder:
     188            yield k
     189    def items(self):
     190        for k in self.keyOrder:
     191            yield k, dict.__getitem__(self, k)
     192    def keys(self):
     193        for k in self.keyOrder:
     194            yield k
     195    def values(self):
     196        for k in self.keyOrder:
     197            yield dict.__getitem__(self, k)
     198    def update(self, dict):
     199        for (k,v) in dict.items():
     200            self.__setitem__(k,v)
    176201
    177202def parse_lookup(kwarg_items, opts):
    178203    # Helper function that handles converting API kwargs (e.g.
    179204    # "name__exact": "tom") to SQL.
    180205
    181     # 'joins' is a dictionary describing the tables that must be joined to complete the query.
     206    # 'joins' is a sorted dictionary describing the tables that must be joined
     207    # to complete the query. The dictionary is sorted because creation order
     208    # is significant; it is a dictionary to ensure uniquess of alias names.
     209    #
    182210    # Each key-value pair follows the form
    183211    #   alias: (table, join_type, condition)
    184212    # where
     
    187215    #   join_type is the type of join (INNER JOIN, LEFT OUTER JOIN, etc)
    188216    #   condition is the where-like statement over which narrows the join.
    189217    #
    190     # alias will be derived from the lookup list name.
    191     # At present, this method only every returns INNER JOINs; the option is there for others
    192     # to implement custom Q()s, etc that return other join types.
    193     tables, joins, where, params = [], {}, [], []
    194     for kwarg, kwarg_value in kwarg_items:
     218        # alias will be derived from the lookup list name.
     219        # At present, this method only every returns INNER JOINs; the option is there for others
     220    # to implement custom Q()s, etc that return other join types.   
     221    tables, joins, where, params = [], SortedDict(), [], []
     222    for kwarg, value in kwarg_items:
    195223        if kwarg in ('order_by', 'limit', 'offset', 'select_related', 'distinct', 'select', 'tables', 'where', 'params'):
    196             continue
    197         if kwarg_value is None:
    198             continue
    199         if kwarg == 'complex':
    200             tables2, joins2, where2, params2 = kwarg_value.get_sql(opts)
     224            pass
     225        elif value is None:
     226            pass
     227        elif kwarg == 'complex':
     228            tables2, joins2, where2, params2 = value.get_sql(opts)
    201229            tables.extend(tables2)
    202230            joins.update(joins2)
    203231            where.extend(where2)
    204232            params.extend(params2)
    205             continue
    206         if kwarg == '_or':
    207             for val in kwarg_value:
     233        elif kwarg == '_or':
     234            for val in value:
    208235                tables2, joins2, where2, params2 = parse_lookup(val, opts)
    209236                tables.extend(tables2)
    210237                joins.update(joins2)
    211238                where.append('(%s)' % ' OR '.join(where2))
    212239                params.extend(params2)
    213             continue
    214         lookup_list = kwarg.split(LOOKUP_SEPARATOR)
    215         # pk="value" is shorthand for (primary key)__exact="value"
    216         if lookup_list[-1] == 'pk':
    217             if opts.pk.rel:
    218                 lookup_list = lookup_list[:-1] + [opts.pk.name, opts.pk.rel.field_name, 'exact']
    219             else:
    220                 lookup_list = lookup_list[:-1] + [opts.pk.name, 'exact']
    221         if len(lookup_list) == 1:
    222             throw_bad_kwarg_error(kwarg)
    223         lookup_type = lookup_list.pop()
    224         current_opts = opts # We'll be overwriting this, so keep a reference to the original opts.
    225         current_table_alias = current_opts.db_table
    226         param_required = False
    227         while lookup_list or param_required:
    228             try:
    229                 # "current" is a piece of the lookup list. For example, in
    230                 # choices.get_list(poll__sites__id__exact=5), lookup_list is
    231                 # ["poll", "sites", "id"], and the first current is "poll".
    232                 try:
    233                     current = lookup_list.pop(0)
    234                 except IndexError:
    235                     # If we're here, lookup_list is empty but param_required
    236                     # is set to True, which means the kwarg was bad.
    237                     # Example: choices.get_list(poll__exact='foo')
    238                     throw_bad_kwarg_error(kwarg)
    239                 # Try many-to-many relationships in the direction in which they are
    240                 # originally defined (i.e., the class that defines the ManyToManyField)
    241                 for f in current_opts.many_to_many:
    242                     if f.name == current:
    243                         rel_table_alias = backend.quote_name("m2m_" + current_table_alias + LOOKUP_SEPARATOR + current)
     240        else: # Must be a search parameter.
     241            path = kwarg.split(LOOKUP_SEPARATOR)
    244242
    245                         joins[rel_table_alias] = (
    246                             backend.quote_name(f.get_m2m_db_table(current_opts)),
    247                             "INNER JOIN",
    248                             '%s.%s = %s.%s' %
    249                                 (backend.quote_name(current_table_alias),
    250                                 backend.quote_name(current_opts.pk.column),
    251                                 rel_table_alias,
    252                                 backend.quote_name(current_opts.object_name.lower() + '_id'))
    253                         )
     243            # Extract the last elements on the kwarg.
     244            # The very last is the clause (equals, like, etc)
     245            # The second last is the table column on which the
     246            # clause is to be performed.
     247            # The only exception to this is pk, which is an implicit
     248            # id__exact; if we find pk, make the clause 'exact', and
     249            # insert a dummy name of None, which we can replace when
     250            # we know which table column to grab as the primary key
     251            clause = path.pop()
     252            if clause == 'pk':
     253                clause = 'exact'
     254                path.append(None)
     255            if len(path) < 1:
     256                raise TypeError, "Cannot parse keyword query '%s'" % kwarg
     257               
     258            tables2, joins2, where2, params2 = lookup_inner(path, clause, value,
     259                                                            opts, opts.db_table, None)
     260            tables.extend(tables2)
     261            joins.update(joins2)
     262            where.extend(where2)
     263            params.extend(params2)
    254264
    255                         # Optimization: In the case of primary-key lookups, we
    256                         # don't have to do an extra join.
    257                         if lookup_list and lookup_list[0] == f.rel.to._meta.pk.name and lookup_type == 'exact':
    258                             where.append(get_where_clause(lookup_type, rel_table_alias+'.',
    259                                 f.rel.to._meta.object_name.lower()+'_id', kwarg_value))
    260                             params.extend(f.get_db_prep_lookup(lookup_type, kwarg_value))
    261                             lookup_list.pop()
    262                             param_required = False
    263                         else:
    264                             new_table_alias = current_table_alias + LOOKUP_SEPARATOR + current
     265    return tables, joins, where, params
    265266
    266                             joins[backend.quote_name(new_table_alias)] = (
    267                                 backend.quote_name(f.rel.to._meta.db_table),
    268                                 "INNER JOIN",
    269                                 '%s.%s = %s.%s' %
    270                                     (rel_table_alias,
    271                                     backend.quote_name(f.rel.to._meta.object_name.lower() + '_id'),
    272                                     backend.quote_name(new_table_alias),
    273                                     backend.quote_name(f.rel.to._meta.pk.column))
    274                             )
    275                             current_table_alias = new_table_alias
    276                             param_required = True
    277                         current_opts = f.rel.to._meta
    278                         raise StopIteration
    279                 # Try many-to-many relationships first in the reverse direction
    280                 # (i.e., from the class does not have the ManyToManyField)
    281                 for f in current_opts.get_all_related_many_to_many_objects():
    282                     if f.name == current:
    283                         rel_table_alias = backend.quote_name("m2m_" + current_table_alias + LOOKUP_SEPARATOR + current)
     267class FieldFound(Exception):
     268    "Exception used to short circuit field-finding operations"
     269    pass
     270   
     271def find_field(name, field_list):
     272    """Find a field with a specific name in a list of field instances.
     273   
     274    Returns None if there are no matches, or several matches
     275   
     276    """
     277    matches = [f for f in field_list if f.name == name]
     278    if len(matches) != 1:
     279        return None
     280    return matches[0]
     281   
     282def lookup_inner(path, clause, value, opts, table, column):
     283    tables, joins, where, params = [], SortedDict(), [], []
     284    current_opts = opts
     285    current_table = table
     286    current_column = column
     287    intermediate_table = None
     288    join_required = False
    284289
    285                         joins[rel_table_alias] = (
    286                             backend.quote_name(f.field.get_m2m_db_table(f.opts)),
    287                             "INNER JOIN",
    288                             '%s.%s = %s.%s' %
    289                                 (backend.quote_name(current_table_alias),
    290                                 backend.quote_name(current_opts.pk.column),
    291                                 rel_table_alias,
    292                                 backend.quote_name(current_opts.object_name.lower() + '_id'))
    293                         )
     290    name = path.pop(0)
     291    # Has the primary key been requested? If so, expand it out
     292    # to be the name of the current class' primary key
     293    if name is None:
     294        name = current_opts.pk.name
     295   
     296    # Try to find the name in the fields associated with the current class
     297    try:                       
     298        # Does the name belong to a defined many-to-many field?
     299        field = find_field(name, current_opts.many_to_many)
     300        if field:
     301            new_table = current_table + "__" + name
     302            new_opts = field.rel.to._meta
     303            new_column = new_opts.pk.column
     304           
     305            # Need to create an intermediate table join over the m2m table
     306            # This process hijacks current_table/column to point to the
     307            # intermediate table.
     308            current_table = "m2m_" + new_table
     309            join_column = new_opts.object_name.lower() + '_id'
     310            intermediate_table = field.get_m2m_db_table(current_opts)
     311           
     312            raise FieldFound()
    294313
    295                         # Optimization: In the case of primary-key lookups, we
    296                         # don't have to do an extra join.
    297                         if lookup_list and lookup_list[0] == f.opts.pk.name and lookup_type == 'exact':
    298                             where.append(get_where_clause(lookup_type, rel_table_alias+'.',
    299                                 f.opts.object_name.lower()+'_id', kwarg_value))
    300                             params.extend(f.field.get_db_prep_lookup(lookup_type, kwarg_value))
    301                             lookup_list.pop()
    302                             param_required = False
    303                         else:
    304                             new_table_alias = current_table_alias + LOOKUP_SEPARATOR + current
     314        # Does the name belong to a reverse defined many-to-many field?
     315        field = find_field(name, current_opts.get_all_related_many_to_many_objects())
     316        if field:           
     317            new_table = current_table + "__" + name
     318            new_opts = field.opts
     319            new_column = new_opts.pk.column
    305320
    306                             joins[backend.quote_name(new_table_alias)] = (
    307                                 backend.quote_name(f.opts.db_table),
    308                                 "INNER JOIN",
    309                                 '%s.%s = %s.%s' %
    310                                     (rel_table_alias,
    311                                     backend.quote_name(f.opts.object_name.lower() + '_id'),
    312                                     backend.quote_name(new_table_alias),
    313                                     backend.quote_name(f.opts.pk.column))
    314                             )
    315                             current_table_alias = new_table_alias
    316                             param_required = True
    317                         current_opts = f.opts
    318                         raise StopIteration
    319                 for f in current_opts.fields:
    320                     # Try many-to-one relationships...
    321                     if f.rel and f.name == current:
    322                         # Optimization: In the case of primary-key lookups, we
    323                         # don't have to do an extra join.
    324                         if lookup_list and lookup_list[0] == f.rel.to._meta.pk.name and lookup_type == 'exact':
    325                             where.append(get_where_clause(lookup_type, current_table_alias+'.', f.column, kwarg_value))
    326                             params.extend(f.get_db_prep_lookup(lookup_type, kwarg_value))
    327                             lookup_list.pop()
    328                             param_required = False
    329                         # 'isnull' lookups in many-to-one relationships are a special case,
    330                         # because we don't want to do a join. We just want to find out
    331                         # whether the foreign key field is NULL.
    332                         elif lookup_type == 'isnull' and not lookup_list:
    333                             where.append(get_where_clause(lookup_type, current_table_alias+'.', f.column, kwarg_value))
    334                             params.extend(f.get_db_prep_lookup(lookup_type, kwarg_value))
    335                         else:
    336                             new_table_alias = current_table_alias + LOOKUP_SEPARATOR + current
     321            # Need to create an intermediate table join over the m2m table.
     322            # This process hijacks current_table/column to point to the
     323            # intermediate table.
     324            current_table = "m2m_" + new_table
     325            join_column = new_opts.object_name.lower() + '_id'
     326            intermediate_table = field.field.get_m2m_db_table(new_opts)
     327           
     328            raise FieldFound()
     329   
     330        # Does the name belong to a one-to-many field?
     331        field = find_field(name, opts.get_all_related_objects())
     332        if field:
     333            new_table = table + "__" + name
     334            new_opts = field.opts
     335            new_column = field.field.column
     336            join_column = opts.pk.column
     337           
     338            # 1-N fields MUST be joined, regardless of any other conditions.
     339            join_required = True
     340           
     341            raise FieldFound()
     342           
     343        # Does the name belong to a one-to-one, many-to-one, or regular field?
     344        field = find_field(name, current_opts.fields)
     345        if field:
     346            if field.rel: # One-to-One/Many-to-one field
     347                new_table = current_table + "__" + name
     348                new_opts = field.rel.to._meta
     349                new_column = new_opts.pk.column
     350                join_column = field.column
     351           
     352            raise FieldFound()
     353           
     354    except FieldFound: # Match found, loop has been shortcut.
     355        pass
     356    except: # Any other exception; rethrow
     357        raise
     358    else: # No match found.
     359        raise TypeError, "Cannot resolve keyword '%s' into field" % name
     360       
    337361
    338                             joins[backend.quote_name(new_table_alias)] = (
    339                                 backend.quote_name(f.rel.to._meta.db_table),
    340                                 "INNER JOIN",
    341                                 '%s.%s = %s.%s' %
    342                                     (backend.quote_name(current_table_alias),
    343                                     backend.quote_name(f.column),
    344                                     backend.quote_name(new_table_alias),
    345                                     backend.quote_name(f.rel.to._meta.pk.column))
    346                             )
    347                             current_table_alias = new_table_alias
    348                             param_required = True
    349                         current_opts = f.rel.to._meta
    350                         raise StopIteration
    351                     # Try direct field-name lookups...
    352                     if f.name == current:
    353                         where.append(get_where_clause(lookup_type, current_table_alias+'.', f.column, kwarg_value))
    354                         params.extend(f.get_db_prep_lookup(lookup_type, kwarg_value))
    355                         param_required = False
    356                         raise StopIteration
    357                 # If we haven't hit StopIteration at this point, "current" must be
    358                 # an invalid lookup, so raise an exception.
    359                 throw_bad_kwarg_error(kwarg)
    360             except StopIteration:
    361                 continue
     362    # Check to see if an intermediate join is required between current_table and new_table
     363    if intermediate_table:
     364        joins[backend.quote_name(current_table)] = (
     365            backend.quote_name(intermediate_table),
     366            "INNER JOIN",
     367            "%s.%s = %s.%s" %
     368                (backend.quote_name(table),
     369                backend.quote_name(current_opts.pk.column),
     370                backend.quote_name(current_table),
     371                backend.quote_name(current_opts.object_name.lower() + '_id'))
     372        )
     373       
     374    if path:
     375        if (len(path) == 1
     376            and path[0] in [new_opts.pk.name, None]
     377            and (clause == 'exact' or  clause == 'isnull')
     378            and not join_required):
     379            # If the last name query is for a key, and the search is for isnull/exact, then
     380            # the current (for N-1) or intermediate (for N-N) table can be used for the search -
     381            # no need to join an extra table just to check the primary key.
     382            new_table = current_table
     383        else:
     384            # There are 1 or more name queries pending, and we have ruled out any short cuts;
     385            # Therefore, a join is required.
     386            joins[backend.quote_name(new_table)] = (
     387                backend.quote_name(new_opts.db_table),
     388                "INNER JOIN",
     389                "%s.%s = %s.%s" %
     390                    (backend.quote_name(current_table),
     391                    backend.quote_name(join_column),
     392                    backend.quote_name(new_table),
     393                    backend.quote_name(new_column))       
     394            )
     395            # If we have made the join, we don't need to tell subsequent
     396            # recursive calls about the column name we joined on.
     397            join_column = None
     398           
     399        # There are name queries remaining. Recurse deeper.
     400        tables2, joins2, where2, params2 = lookup_inner(path, clause, value,
     401                                                        new_opts, new_table, join_column)
     402
     403        tables.extend(tables2)
     404        joins.update(joins2)
     405        where.extend(where2)
     406        params.extend(params2)   
     407    else:
     408        # Evaluate clause on current table.
     409        if (name in [current_opts.pk.name, None]
     410            and (clause == 'exact' or clause == 'isnull')
     411            and current_column):
     412            # If this is an exact/isnull key search, and the last pass found/introduced
     413            # a current/intermediate table that we can use to optimize the query,
     414            # then use that column name
     415            column = current_column
     416        else:
     417            column = field.column
     418
     419        where.append(get_where_clause(clause, current_table + '.', column, value))
     420        params.extend(field.get_db_prep_lookup(clause, value))
     421
    362422    return tables, joins, where, params
Back to Top