Ticket #7231: extra-join-1.2-svn-12906.diff

File extra-join-1.2-svn-12906.diff, 123.8 KB (added by gordyt, 5 years ago)

Updated for Django version 1.2, svn build number 12906

  • django/db/models/query.py

    diff -rupN ../django-trunk/django/db/models/query.py django/db/models/query.py
    old new class QuerySet(object): 
    648648        return obj
    649649
    650650    def extra(self, select=None, where=None, params=None, tables=None,
    651               order_by=None, select_params=None):
     651              order_by=None, select_params=None, join=None):
    652652        """
    653653        Adds extra SQL fragments to the query.
    654654        """
    655655        assert self.query.can_filter(), \
    656656                "Cannot change a query once a slice has been taken"
    657657        clone = self._clone()
    658         clone.query.add_extra(select, select_params, where, params, tables, order_by)
     658        clone.query.add_extra(select, select_params, where, params, tables, order_by, join)
    659659        return clone
    660660
    661661    def reverse(self):
  • django/db/models/sql/compiler.py

    diff -rupN ../django-trunk/django/db/models/sql/compiler.py django/db/models/sql/compiler.py
    old new class SQLCompiler(object): 
    7777        result.extend(from_)
    7878        params.extend(f_params)
    7979
     80        if self.query.extra_join:
     81            result.append(' '.join(self.query.extra_join))
     82
    8083        if where:
    8184            result.append('WHERE %s' % where)
    8285            params.extend(w_params)
  • django/db/models/sql/compiler.py.orig

    diff -rupN ../django-trunk/django/db/models/sql/compiler.py.orig django/db/models/sql/compiler.py.orig
    old new  
     1from django.core.exceptions import FieldError
     2from django.db import connections
     3from django.db.backends.util import truncate_name
     4from django.db.models.sql.constants import *
     5from django.db.models.sql.datastructures import EmptyResultSet
     6from django.db.models.sql.expressions import SQLEvaluator
     7from django.db.models.sql.query import get_proxied_model, get_order_dir, \
     8     select_related_descend, Query
     9
     10class SQLCompiler(object):
     11    def __init__(self, query, connection, using):
     12        self.query = query
     13        self.connection = connection
     14        self.using = using
     15        self.quote_cache = {}
     16
     17    def pre_sql_setup(self):
     18        """
     19        Does any necessary class setup immediately prior to producing SQL. This
     20        is for things that can't necessarily be done in __init__ because we
     21        might not have all the pieces in place at that time.
     22        """
     23        if not self.query.tables:
     24            self.query.join((None, self.query.model._meta.db_table, None, None))
     25        if (not self.query.select and self.query.default_cols and not
     26                self.query.included_inherited_models):
     27            self.query.setup_inherited_models()
     28        if self.query.select_related and not self.query.related_select_cols:
     29            self.fill_related_selections()
     30
     31    def quote_name_unless_alias(self, name):
     32        """
     33        A wrapper around connection.ops.quote_name that doesn't quote aliases
     34        for table names. This avoids problems with some SQL dialects that treat
     35        quoted strings specially (e.g. PostgreSQL).
     36        """
     37        if name in self.quote_cache:
     38            return self.quote_cache[name]
     39        if ((name in self.query.alias_map and name not in self.query.table_map) or
     40                name in self.query.extra_select):
     41            self.quote_cache[name] = name
     42            return name
     43        r = self.connection.ops.quote_name(name)
     44        self.quote_cache[name] = r
     45        return r
     46
     47    def as_sql(self, with_limits=True, with_col_aliases=False):
     48        """
     49        Creates the SQL for this query. Returns the SQL string and list of
     50        parameters.
     51
     52        If 'with_limits' is False, any limit/offset information is not included
     53        in the query.
     54        """
     55        self.pre_sql_setup()
     56        out_cols = self.get_columns(with_col_aliases)
     57        ordering, ordering_group_by = self.get_ordering()
     58
     59        # This must come after 'select' and 'ordering' -- see docstring of
     60        # get_from_clause() for details.
     61        from_, f_params = self.get_from_clause()
     62
     63        qn = self.quote_name_unless_alias
     64
     65        where, w_params = self.query.where.as_sql(qn=qn, connection=self.connection)
     66        having, h_params = self.query.having.as_sql(qn=qn, connection=self.connection)
     67        params = []
     68        for val in self.query.extra_select.itervalues():
     69            params.extend(val[1])
     70
     71        result = ['SELECT']
     72        if self.query.distinct:
     73            result.append('DISTINCT')
     74        result.append(', '.join(out_cols + self.query.ordering_aliases))
     75
     76        result.append('FROM')
     77        result.extend(from_)
     78        params.extend(f_params)
     79
     80        if where:
     81            result.append('WHERE %s' % where)
     82            params.extend(w_params)
     83
     84        grouping, gb_params = self.get_grouping()
     85        if grouping:
     86            if ordering:
     87                # If the backend can't group by PK (i.e., any database
     88                # other than MySQL), then any fields mentioned in the
     89                # ordering clause needs to be in the group by clause.
     90                if not self.connection.features.allows_group_by_pk:
     91                    for col, col_params in ordering_group_by:
     92                        if col not in grouping:
     93                            grouping.append(str(col))
     94                            gb_params.extend(col_params)
     95            else:
     96                ordering = self.connection.ops.force_no_ordering()
     97            result.append('GROUP BY %s' % ', '.join(grouping))
     98            params.extend(gb_params)
     99
     100        if having:
     101            result.append('HAVING %s' % having)
     102            params.extend(h_params)
     103
     104        if ordering:
     105            result.append('ORDER BY %s' % ', '.join(ordering))
     106
     107        if with_limits:
     108            if self.query.high_mark is not None:
     109                result.append('LIMIT %d' % (self.query.high_mark - self.query.low_mark))
     110            if self.query.low_mark:
     111                if self.query.high_mark is None:
     112                    val = self.connection.ops.no_limit_value()
     113                    if val:
     114                        result.append('LIMIT %d' % val)
     115                result.append('OFFSET %d' % self.query.low_mark)
     116
     117        return ' '.join(result), tuple(params)
     118
     119    def as_nested_sql(self):
     120        """
     121        Perform the same functionality as the as_sql() method, returning an
     122        SQL string and parameters. However, the alias prefixes are bumped
     123        beforehand (in a copy -- the current query isn't changed) and any
     124        ordering is removed.
     125
     126        Used when nesting this query inside another.
     127        """
     128        obj = self.query.clone()
     129        obj.clear_ordering(True)
     130        obj.bump_prefix()
     131        return obj.get_compiler(connection=self.connection).as_sql()
     132
     133    def get_columns(self, with_aliases=False):
     134        """
     135        Returns the list of columns to use in the select statement. If no
     136        columns have been specified, returns all columns relating to fields in
     137        the model.
     138
     139        If 'with_aliases' is true, any column names that are duplicated
     140        (without the table names) are given unique aliases. This is needed in
     141        some cases to avoid ambiguity with nested queries.
     142        """
     143        qn = self.quote_name_unless_alias
     144        qn2 = self.connection.ops.quote_name
     145        result = ['(%s) AS %s' % (col[0], qn2(alias)) for alias, col in self.query.extra_select.iteritems()]
     146        aliases = set(self.query.extra_select.keys())
     147        if with_aliases:
     148            col_aliases = aliases.copy()
     149        else:
     150            col_aliases = set()
     151        if self.query.select:
     152            only_load = self.deferred_to_columns()
     153            for col in self.query.select:
     154                if isinstance(col, (list, tuple)):
     155                    alias, column = col
     156                    table = self.query.alias_map[alias][TABLE_NAME]
     157                    if table in only_load and col not in only_load[table]:
     158                        continue
     159                    r = '%s.%s' % (qn(alias), qn(column))
     160                    if with_aliases:
     161                        if col[1] in col_aliases:
     162                            c_alias = 'Col%d' % len(col_aliases)
     163                            result.append('%s AS %s' % (r, c_alias))
     164                            aliases.add(c_alias)
     165                            col_aliases.add(c_alias)
     166                        else:
     167                            result.append('%s AS %s' % (r, qn2(col[1])))
     168                            aliases.add(r)
     169                            col_aliases.add(col[1])
     170                    else:
     171                        result.append(r)
     172                        aliases.add(r)
     173                        col_aliases.add(col[1])
     174                else:
     175                    result.append(col.as_sql(qn, self.connection))
     176
     177                    if hasattr(col, 'alias'):
     178                        aliases.add(col.alias)
     179                        col_aliases.add(col.alias)
     180
     181        elif self.query.default_cols:
     182            cols, new_aliases = self.get_default_columns(with_aliases,
     183                    col_aliases)
     184            result.extend(cols)
     185            aliases.update(new_aliases)
     186
     187        max_name_length = self.connection.ops.max_name_length()
     188        result.extend([
     189            '%s%s' % (
     190                aggregate.as_sql(qn, self.connection),
     191                alias is not None
     192                    and ' AS %s' % qn(truncate_name(alias, max_name_length))
     193                    or ''
     194            )
     195            for alias, aggregate in self.query.aggregate_select.items()
     196        ])
     197
     198        for table, col in self.query.related_select_cols:
     199            r = '%s.%s' % (qn(table), qn(col))
     200            if with_aliases and col in col_aliases:
     201                c_alias = 'Col%d' % len(col_aliases)
     202                result.append('%s AS %s' % (r, c_alias))
     203                aliases.add(c_alias)
     204                col_aliases.add(c_alias)
     205            else:
     206                result.append(r)
     207                aliases.add(r)
     208                col_aliases.add(col)
     209
     210        self._select_aliases = aliases
     211        return result
     212
     213    def get_default_columns(self, with_aliases=False, col_aliases=None,
     214            start_alias=None, opts=None, as_pairs=False, local_only=False):
     215        """
     216        Computes the default columns for selecting every field in the base
     217        model. Will sometimes be called to pull in related models (e.g. via
     218        select_related), in which case "opts" and "start_alias" will be given
     219        to provide a starting point for the traversal.
     220
     221        Returns a list of strings, quoted appropriately for use in SQL
     222        directly, as well as a set of aliases used in the select statement (if
     223        'as_pairs' is True, returns a list of (alias, col_name) pairs instead
     224        of strings as the first component and None as the second component).
     225        """
     226        result = []
     227        if opts is None:
     228            opts = self.query.model._meta
     229        qn = self.quote_name_unless_alias
     230        qn2 = self.connection.ops.quote_name
     231        aliases = set()
     232        only_load = self.deferred_to_columns()
     233        # Skip all proxy to the root proxied model
     234        proxied_model = get_proxied_model(opts)
     235
     236        if start_alias:
     237            seen = {None: start_alias}
     238        for field, model in opts.get_fields_with_model():
     239            if local_only and model is not None:
     240                continue
     241            if start_alias:
     242                try:
     243                    alias = seen[model]
     244                except KeyError:
     245                    if model is proxied_model:
     246                        alias = start_alias
     247                    else:
     248                        link_field = opts.get_ancestor_link(model)
     249                        alias = self.query.join((start_alias, model._meta.db_table,
     250                                link_field.column, model._meta.pk.column))
     251                    seen[model] = alias
     252            else:
     253                # If we're starting from the base model of the queryset, the
     254                # aliases will have already been set up in pre_sql_setup(), so
     255                # we can save time here.
     256                alias = self.query.included_inherited_models[model]
     257            table = self.query.alias_map[alias][TABLE_NAME]
     258            if table in only_load and field.column not in only_load[table]:
     259                continue
     260            if as_pairs:
     261                result.append((alias, field.column))
     262                aliases.add(alias)
     263                continue
     264            if with_aliases and field.column in col_aliases:
     265                c_alias = 'Col%d' % len(col_aliases)
     266                result.append('%s.%s AS %s' % (qn(alias),
     267                    qn2(field.column), c_alias))
     268                col_aliases.add(c_alias)
     269                aliases.add(c_alias)
     270            else:
     271                r = '%s.%s' % (qn(alias), qn2(field.column))
     272                result.append(r)
     273                aliases.add(r)
     274                if with_aliases:
     275                    col_aliases.add(field.column)
     276        return result, aliases
     277
     278    def get_ordering(self):
     279        """
     280        Returns a tuple containing a list representing the SQL elements in the
     281        "order by" clause, and the list of SQL elements that need to be added
     282        to the GROUP BY clause as a result of the ordering.
     283
     284        Also sets the ordering_aliases attribute on this instance to a list of
     285        extra aliases needed in the select.
     286
     287        Determining the ordering SQL can change the tables we need to include,
     288        so this should be run *before* get_from_clause().
     289        """
     290        if self.query.extra_order_by:
     291            ordering = self.query.extra_order_by
     292        elif not self.query.default_ordering:
     293            ordering = self.query.order_by
     294        else:
     295            ordering = self.query.order_by or self.query.model._meta.ordering
     296        qn = self.quote_name_unless_alias
     297        qn2 = self.connection.ops.quote_name
     298        distinct = self.query.distinct
     299        select_aliases = self._select_aliases
     300        result = []
     301        group_by = []
     302        ordering_aliases = []
     303        if self.query.standard_ordering:
     304            asc, desc = ORDER_DIR['ASC']
     305        else:
     306            asc, desc = ORDER_DIR['DESC']
     307
     308        # It's possible, due to model inheritance, that normal usage might try
     309        # to include the same field more than once in the ordering. We track
     310        # the table/column pairs we use and discard any after the first use.
     311        processed_pairs = set()
     312
     313        for field in ordering:
     314            if field == '?':
     315                result.append(self.connection.ops.random_function_sql())
     316                continue
     317            if isinstance(field, int):
     318                if field < 0:
     319                    order = desc
     320                    field = -field
     321                else:
     322                    order = asc
     323                result.append('%s %s' % (field, order))
     324                group_by.append((field, []))
     325                continue
     326            col, order = get_order_dir(field, asc)
     327            if col in self.query.aggregate_select:
     328                result.append('%s %s' % (col, order))
     329                continue
     330            if '.' in field:
     331                # This came in through an extra(order_by=...) addition. Pass it
     332                # on verbatim.
     333                table, col = col.split('.', 1)
     334                if (table, col) not in processed_pairs:
     335                    elt = '%s.%s' % (qn(table), col)
     336                    processed_pairs.add((table, col))
     337                    if not distinct or elt in select_aliases:
     338                        result.append('%s %s' % (elt, order))
     339                        group_by.append((elt, []))
     340            elif get_order_dir(field)[0] not in self.query.extra_select:
     341                # 'col' is of the form 'field' or 'field1__field2' or
     342                # '-field1__field2__field', etc.
     343                for table, col, order in self.find_ordering_name(field,
     344                        self.query.model._meta, default_order=asc):
     345                    if (table, col) not in processed_pairs:
     346                        elt = '%s.%s' % (qn(table), qn2(col))
     347                        processed_pairs.add((table, col))
     348                        if distinct and elt not in select_aliases:
     349                            ordering_aliases.append(elt)
     350                        result.append('%s %s' % (elt, order))
     351                        group_by.append((elt, []))
     352            else:
     353                elt = qn2(col)
     354                if distinct and col not in select_aliases:
     355                    ordering_aliases.append(elt)
     356                result.append('%s %s' % (elt, order))
     357                group_by.append(self.query.extra_select[col])
     358        self.query.ordering_aliases = ordering_aliases
     359        return result, group_by
     360
     361    def find_ordering_name(self, name, opts, alias=None, default_order='ASC',
     362            already_seen=None):
     363        """
     364        Returns the table alias (the name might be ambiguous, the alias will
     365        not be) and column name for ordering by the given 'name' parameter.
     366        The 'name' is of the form 'field1__field2__...__fieldN'.
     367        """
     368        name, order = get_order_dir(name, default_order)
     369        pieces = name.split(LOOKUP_SEP)
     370        if not alias:
     371            alias = self.query.get_initial_alias()
     372        field, target, opts, joins, last, extra = self.query.setup_joins(pieces,
     373                opts, alias, False)
     374        alias = joins[-1]
     375        col = target.column
     376        if not field.rel:
     377            # To avoid inadvertent trimming of a necessary alias, use the
     378            # refcount to show that we are referencing a non-relation field on
     379            # the model.
     380            self.query.ref_alias(alias)
     381
     382        # Must use left outer joins for nullable fields and their relations.
     383        self.query.promote_alias_chain(joins,
     384            self.query.alias_map[joins[0]][JOIN_TYPE] == self.query.LOUTER)
     385
     386        # If we get to this point and the field is a relation to another model,
     387        # append the default ordering for that model.
     388        if field.rel and len(joins) > 1 and opts.ordering:
     389            # Firstly, avoid infinite loops.
     390            if not already_seen:
     391                already_seen = set()
     392            join_tuple = tuple([self.query.alias_map[j][TABLE_NAME] for j in joins])
     393            if join_tuple in already_seen:
     394                raise FieldError('Infinite loop caused by ordering.')
     395            already_seen.add(join_tuple)
     396
     397            results = []
     398            for item in opts.ordering:
     399                results.extend(self.find_ordering_name(item, opts, alias,
     400                        order, already_seen))
     401            return results
     402
     403        if alias:
     404            # We have to do the same "final join" optimisation as in
     405            # add_filter, since the final column might not otherwise be part of
     406            # the select set (so we can't order on it).
     407            while 1:
     408                join = self.query.alias_map[alias]
     409                if col != join[RHS_JOIN_COL]:
     410                    break
     411                self.query.unref_alias(alias)
     412                alias = join[LHS_ALIAS]
     413                col = join[LHS_JOIN_COL]
     414        return [(alias, col, order)]
     415
     416    def get_from_clause(self):
     417        """
     418        Returns a list of strings that are joined together to go after the
     419        "FROM" part of the query, as well as a list any extra parameters that
     420        need to be included. Sub-classes, can override this to create a
     421        from-clause via a "select".
     422
     423        This should only be called after any SQL construction methods that
     424        might change the tables we need. This means the select columns and
     425        ordering must be done first.
     426        """
     427        result = []
     428        qn = self.quote_name_unless_alias
     429        qn2 = self.connection.ops.quote_name
     430        first = True
     431        for alias in self.query.tables:
     432            if not self.query.alias_refcount[alias]:
     433                continue
     434            try:
     435                name, alias, join_type, lhs, lhs_col, col, nullable = self.query.alias_map[alias]
     436            except KeyError:
     437                # Extra tables can end up in self.tables, but not in the
     438                # alias_map if they aren't in a join. That's OK. We skip them.
     439                continue
     440            alias_str = (alias != name and ' %s' % alias or '')
     441            if join_type and not first:
     442                result.append('%s %s%s ON (%s.%s = %s.%s)'
     443                        % (join_type, qn(name), alias_str, qn(lhs),
     444                           qn2(lhs_col), qn(alias), qn2(col)))
     445            else:
     446                connector = not first and ', ' or ''
     447                result.append('%s%s%s' % (connector, qn(name), alias_str))
     448            first = False
     449        for t in self.query.extra_tables:
     450            alias, unused = self.query.table_alias(t)
     451            # Only add the alias if it's not already present (the table_alias()
     452            # calls increments the refcount, so an alias refcount of one means
     453            # this is the only reference.
     454            if alias not in self.query.alias_map or self.query.alias_refcount[alias] == 1:
     455                connector = not first and ', ' or ''
     456                result.append('%s%s' % (connector, qn(alias)))
     457                first = False
     458        return result, []
     459
     460    def get_grouping(self):
     461        """
     462        Returns a tuple representing the SQL elements in the "group by" clause.
     463        """
     464        qn = self.quote_name_unless_alias
     465        result, params = [], []
     466        if self.query.group_by is not None:
     467            if len(self.query.model._meta.fields) == len(self.query.select) and \
     468                self.connection.features.allows_group_by_pk:
     469                self.query.group_by = [(self.query.model._meta.db_table, self.query.model._meta.pk.column)]
     470
     471            group_by = self.query.group_by or []
     472
     473            extra_selects = []
     474            for extra_select, extra_params in self.query.extra_select.itervalues():
     475                extra_selects.append(extra_select)
     476                params.extend(extra_params)
     477            for col in group_by + self.query.related_select_cols + extra_selects:
     478                if isinstance(col, (list, tuple)):
     479                    result.append('%s.%s' % (qn(col[0]), qn(col[1])))
     480                elif hasattr(col, 'as_sql'):
     481                    result.append(col.as_sql(qn))
     482                else:
     483                    result.append('(%s)' % str(col))
     484        return result, params
     485
     486    def fill_related_selections(self, opts=None, root_alias=None, cur_depth=1,
     487            used=None, requested=None, restricted=None, nullable=None,
     488            dupe_set=None, avoid_set=None):
     489        """
     490        Fill in the information needed for a select_related query. The current
     491        depth is measured as the number of connections away from the root model
     492        (for example, cur_depth=1 means we are looking at models with direct
     493        connections to the root model).
     494        """
     495        if not restricted and self.query.max_depth and cur_depth > self.query.max_depth:
     496            # We've recursed far enough; bail out.
     497            return
     498
     499        if not opts:
     500            opts = self.query.get_meta()
     501            root_alias = self.query.get_initial_alias()
     502            self.query.related_select_cols = []
     503            self.query.related_select_fields = []
     504        if not used:
     505            used = set()
     506        if dupe_set is None:
     507            dupe_set = set()
     508        if avoid_set is None:
     509            avoid_set = set()
     510        orig_dupe_set = dupe_set
     511
     512        # Setup for the case when only particular related fields should be
     513        # included in the related selection.
     514        if requested is None:
     515            if isinstance(self.query.select_related, dict):
     516                requested = self.query.select_related
     517                restricted = True
     518            else:
     519                restricted = False
     520
     521        for f, model in opts.get_fields_with_model():
     522            if not select_related_descend(f, restricted, requested):
     523                continue
     524            # The "avoid" set is aliases we want to avoid just for this
     525            # particular branch of the recursion. They aren't permanently
     526            # forbidden from reuse in the related selection tables (which is
     527            # what "used" specifies).
     528            avoid = avoid_set.copy()
     529            dupe_set = orig_dupe_set.copy()
     530            table = f.rel.to._meta.db_table
     531            promote = nullable or f.null
     532            if model:
     533                int_opts = opts
     534                alias = root_alias
     535                alias_chain = []
     536                for int_model in opts.get_base_chain(model):
     537                    # Proxy model have elements in base chain
     538                    # with no parents, assign the new options
     539                    # object and skip to the next base in that
     540                    # case
     541                    if not int_opts.parents[int_model]:
     542                        int_opts = int_model._meta
     543                        continue
     544                    lhs_col = int_opts.parents[int_model].column
     545                    dedupe = lhs_col in opts.duplicate_targets
     546                    if dedupe:
     547                        avoid.update(self.query.dupe_avoidance.get(id(opts), lhs_col),
     548                                ())
     549                        dupe_set.add((opts, lhs_col))
     550                    int_opts = int_model._meta
     551                    alias = self.query.join((alias, int_opts.db_table, lhs_col,
     552                            int_opts.pk.column), exclusions=used,
     553                            promote=promote)
     554                    alias_chain.append(alias)
     555                    for (dupe_opts, dupe_col) in dupe_set:
     556                        self.query.update_dupe_avoidance(dupe_opts, dupe_col, alias)
     557                if self.query.alias_map[root_alias][JOIN_TYPE] == self.query.LOUTER:
     558                    self.query.promote_alias_chain(alias_chain, True)
     559            else:
     560                alias = root_alias
     561
     562            dedupe = f.column in opts.duplicate_targets
     563            if dupe_set or dedupe:
     564                avoid.update(self.query.dupe_avoidance.get((id(opts), f.column), ()))
     565                if dedupe:
     566                    dupe_set.add((opts, f.column))
     567
     568            alias = self.query.join((alias, table, f.column,
     569                    f.rel.get_related_field().column),
     570                    exclusions=used.union(avoid), promote=promote)
     571            used.add(alias)
     572            columns, aliases = self.get_default_columns(start_alias=alias,
     573                    opts=f.rel.to._meta, as_pairs=True)
     574            self.query.related_select_cols.extend(columns)
     575            if self.query.alias_map[alias][JOIN_TYPE] == self.query.LOUTER:
     576                self.query.promote_alias_chain(aliases, True)
     577            self.query.related_select_fields.extend(f.rel.to._meta.fields)
     578            if restricted:
     579                next = requested.get(f.name, {})
     580            else:
     581                next = False
     582            new_nullable = f.null or promote
     583            for dupe_opts, dupe_col in dupe_set:
     584                self.query.update_dupe_avoidance(dupe_opts, dupe_col, alias)
     585            self.fill_related_selections(f.rel.to._meta, alias, cur_depth + 1,
     586                    used, next, restricted, new_nullable, dupe_set, avoid)
     587
     588        if restricted:
     589            related_fields = [
     590                (o.field, o.model)
     591                for o in opts.get_all_related_objects()
     592                if o.field.unique
     593            ]
     594            for f, model in related_fields:
     595                if not select_related_descend(f, restricted, requested, reverse=True):
     596                    continue
     597                # The "avoid" set is aliases we want to avoid just for this
     598                # particular branch of the recursion. They aren't permanently
     599                # forbidden from reuse in the related selection tables (which is
     600                # what "used" specifies).
     601                avoid = avoid_set.copy()
     602                dupe_set = orig_dupe_set.copy()
     603                table = model._meta.db_table
     604
     605                int_opts = opts
     606                alias = root_alias
     607                alias_chain = []
     608                chain = opts.get_base_chain(f.rel.to)
     609                if chain is not None:
     610                    for int_model in chain:
     611                        # Proxy model have elements in base chain
     612                        # with no parents, assign the new options
     613                        # object and skip to the next base in that
     614                        # case
     615                        if not int_opts.parents[int_model]:
     616                            int_opts = int_model._meta
     617                            continue
     618                        lhs_col = int_opts.parents[int_model].column
     619                        dedupe = lhs_col in opts.duplicate_targets
     620                        if dedupe:
     621                            avoid.update(self.query.dupe_avoidance.get(id(opts), lhs_col),
     622                                ())
     623                            dupe_set.add((opts, lhs_col))
     624                        int_opts = int_model._meta
     625                        alias = self.query.join(
     626                            (alias, int_opts.db_table, lhs_col, int_opts.pk.column),
     627                            exclusions=used, promote=True, reuse=used
     628                        )
     629                        alias_chain.append(alias)
     630                        for dupe_opts, dupe_col in dupe_set:
     631                            self.query.update_dupe_avoidance(dupe_opts, dupe_col, alias)
     632                    dedupe = f.column in opts.duplicate_targets
     633                    if dupe_set or dedupe:
     634                        avoid.update(self.query.dupe_avoidance.get((id(opts), f.column), ()))
     635                        if dedupe:
     636                            dupe_set.add((opts, f.column))
     637                alias = self.query.join(
     638                    (alias, table, f.rel.get_related_field().column, f.column),
     639                    exclusions=used.union(avoid),
     640                    promote=True
     641                )
     642                used.add(alias)
     643                columns, aliases = self.get_default_columns(start_alias=alias,
     644                    opts=model._meta, as_pairs=True, local_only=True)
     645                self.query.related_select_cols.extend(columns)
     646                self.query.related_select_fields.extend(model._meta.fields)
     647
     648                next = requested.get(f.related_query_name(), {})
     649                new_nullable = f.null or None
     650
     651                self.fill_related_selections(model._meta, table, cur_depth+1,
     652                    used, next, restricted, new_nullable)
     653
     654    def deferred_to_columns(self):
     655        """
     656        Converts the self.deferred_loading data structure to mapping of table
     657        names to sets of column names which are to be loaded. Returns the
     658        dictionary.
     659        """
     660        columns = {}
     661        self.query.deferred_to_data(columns, self.query.deferred_to_columns_cb)
     662        return columns
     663
     664    def results_iter(self):
     665        """
     666        Returns an iterator over the results from executing this query.
     667        """
     668        resolve_columns = hasattr(self, 'resolve_columns')
     669        fields = None
     670        for rows in self.execute_sql(MULTI):
     671            for row in rows:
     672                if resolve_columns:
     673                    if fields is None:
     674                        # We only set this up here because
     675                        # related_select_fields isn't populated until
     676                        # execute_sql() has been called.
     677                        if self.query.select_fields:
     678                            fields = self.query.select_fields + self.query.related_select_fields
     679                        else:
     680                            fields = self.query.model._meta.fields
     681                        # If the field was deferred, exclude it from being passed
     682                        # into `resolve_columns` because it wasn't selected.
     683                        only_load = self.deferred_to_columns()
     684                        if only_load:
     685                            db_table = self.query.model._meta.db_table
     686                            fields = [f for f in fields if db_table in only_load and
     687                                      f.column in only_load[db_table]]
     688                    row = self.resolve_columns(row, fields)
     689
     690                if self.query.aggregate_select:
     691                    aggregate_start = len(self.query.extra_select.keys()) + len(self.query.select)
     692                    aggregate_end = aggregate_start + len(self.query.aggregate_select)
     693                    row = tuple(row[:aggregate_start]) + tuple([
     694                        self.query.resolve_aggregate(value, aggregate, self.connection)
     695                        for (alias, aggregate), value
     696                        in zip(self.query.aggregate_select.items(), row[aggregate_start:aggregate_end])
     697                    ]) + tuple(row[aggregate_end:])
     698
     699                yield row
     700
     701    def execute_sql(self, result_type=MULTI):
     702        """
     703        Run the query against the database and returns the result(s). The
     704        return value is a single data item if result_type is SINGLE, or an
     705        iterator over the results if the result_type is MULTI.
     706
     707        result_type is either MULTI (use fetchmany() to retrieve all rows),
     708        SINGLE (only retrieve a single row), or None. In this last case, the
     709        cursor is returned if any query is executed, since it's used by
     710        subclasses such as InsertQuery). It's possible, however, that no query
     711        is needed, as the filters describe an empty set. In that case, None is
     712        returned, to avoid any unnecessary database interaction.
     713        """
     714        try:
     715            sql, params = self.as_sql()
     716            if not sql:
     717                raise EmptyResultSet
     718        except EmptyResultSet:
     719            if result_type == MULTI:
     720                return empty_iter()
     721            else:
     722                return
     723
     724        cursor = self.connection.cursor()
     725        cursor.execute(sql, params)
     726
     727        if not result_type:
     728            return cursor
     729        if result_type == SINGLE:
     730            if self.query.ordering_aliases:
     731                return cursor.fetchone()[:-len(self.query.ordering_aliases)]
     732            return cursor.fetchone()
     733
     734        # The MULTI case.
     735        if self.query.ordering_aliases:
     736            result = order_modified_iter(cursor, len(self.query.ordering_aliases),
     737                    self.connection.features.empty_fetchmany_value)
     738        else:
     739            result = iter((lambda: cursor.fetchmany(GET_ITERATOR_CHUNK_SIZE)),
     740                    self.connection.features.empty_fetchmany_value)
     741        if not self.connection.features.can_use_chunked_reads:
     742            # If we are using non-chunked reads, we return the same data
     743            # structure as normally, but ensure it is all read into memory
     744            # before going any further.
     745            return list(result)
     746        return result
     747
     748
     749class SQLInsertCompiler(SQLCompiler):
     750    def placeholder(self, field, val):
     751        if field is None:
     752            # A field value of None means the value is raw.
     753            return val
     754        elif hasattr(field, 'get_placeholder'):
     755            # Some fields (e.g. geo fields) need special munging before
     756            # they can be inserted.
     757            return field.get_placeholder(val, self.connection)
     758        else:
     759            # Return the common case for the placeholder
     760            return '%s'
     761
     762    def as_sql(self):
     763        # We don't need quote_name_unless_alias() here, since these are all
     764        # going to be column names (so we can avoid the extra overhead).
     765        qn = self.connection.ops.quote_name
     766        opts = self.query.model._meta
     767        result = ['INSERT INTO %s' % qn(opts.db_table)]
     768        result.append('(%s)' % ', '.join([qn(c) for c in self.query.columns]))
     769        values = [self.placeholder(*v) for v in self.query.values]
     770        result.append('VALUES (%s)' % ', '.join(values))
     771        params = self.query.params
     772        if self.return_id and self.connection.features.can_return_id_from_insert:
     773            col = "%s.%s" % (qn(opts.db_table), qn(opts.pk.column))
     774            r_fmt, r_params = self.connection.ops.return_insert_id()
     775            result.append(r_fmt % col)
     776            params = params + r_params
     777        return ' '.join(result), params
     778
     779    def execute_sql(self, return_id=False):
     780        self.return_id = return_id
     781        cursor = super(SQLInsertCompiler, self).execute_sql(None)
     782        if not (return_id and cursor):
     783            return
     784        if self.connection.features.can_return_id_from_insert:
     785            return self.connection.ops.fetch_returned_insert_id(cursor)
     786        return self.connection.ops.last_insert_id(cursor,
     787                self.query.model._meta.db_table, self.query.model._meta.pk.column)
     788
     789
     790class SQLDeleteCompiler(SQLCompiler):
     791    def as_sql(self):
     792        """
     793        Creates the SQL for this query. Returns the SQL string and list of
     794        parameters.
     795        """
     796        assert len(self.query.tables) == 1, \
     797                "Can only delete from one table at a time."
     798        qn = self.quote_name_unless_alias
     799        result = ['DELETE FROM %s' % qn(self.query.tables[0])]
     800        where, params = self.query.where.as_sql(qn=qn, connection=self.connection)
     801        result.append('WHERE %s' % where)
     802        return ' '.join(result), tuple(params)
     803
     804class SQLUpdateCompiler(SQLCompiler):
     805    def as_sql(self):
     806        """
     807        Creates the SQL for this query. Returns the SQL string and list of
     808        parameters.
     809        """
     810        from django.db.models.base import Model
     811
     812        self.pre_sql_setup()
     813        if not self.query.values:
     814            return '', ()
     815        table = self.query.tables[0]
     816        qn = self.quote_name_unless_alias
     817        result = ['UPDATE %s' % qn(table)]
     818        result.append('SET')
     819        values, update_params = [], []
     820        for field, model, val in self.query.values:
     821            if hasattr(val, 'prepare_database_save'):
     822                val = val.prepare_database_save(field)
     823            else:
     824                val = field.get_db_prep_save(val, connection=self.connection)
     825
     826            # Getting the placeholder for the field.
     827            if hasattr(field, 'get_placeholder'):
     828                placeholder = field.get_placeholder(val, self.connection)
     829            else:
     830                placeholder = '%s'
     831
     832            if hasattr(val, 'evaluate'):
     833                val = SQLEvaluator(val, self.query, allow_joins=False)
     834            name = field.column
     835            if hasattr(val, 'as_sql'):
     836                sql, params = val.as_sql(qn, self.connection)
     837                values.append('%s = %s' % (qn(name), sql))
     838                update_params.extend(params)
     839            elif val is not None:
     840                values.append('%s = %s' % (qn(name), placeholder))
     841                update_params.append(val)
     842            else:
     843                values.append('%s = NULL' % qn(name))
     844        if not values:
     845            return '', ()
     846        result.append(', '.join(values))
     847        where, params = self.query.where.as_sql(qn=qn, connection=self.connection)
     848        if where:
     849            result.append('WHERE %s' % where)
     850        return ' '.join(result), tuple(update_params + params)
     851
     852    def execute_sql(self, result_type):
     853        """
     854        Execute the specified update. Returns the number of rows affected by
     855        the primary update query. The "primary update query" is the first
     856        non-empty query that is executed. Row counts for any subsequent,
     857        related queries are not available.
     858        """
     859        cursor = super(SQLUpdateCompiler, self).execute_sql(result_type)
     860        rows = cursor and cursor.rowcount or 0
     861        is_empty = cursor is None
     862        del cursor
     863        for query in self.query.get_related_updates():
     864            aux_rows = query.get_compiler(self.using).execute_sql(result_type)
     865            if is_empty:
     866                rows = aux_rows
     867                is_empty = False
     868        return rows
     869
     870    def pre_sql_setup(self):
     871        """
     872        If the update depends on results from other tables, we need to do some
     873        munging of the "where" conditions to match the format required for
     874        (portable) SQL updates. That is done here.
     875
     876        Further, if we are going to be running multiple updates, we pull out
     877        the id values to update at this point so that they don't change as a
     878        result of the progressive updates.
     879        """
     880        self.query.select_related = False
     881        self.query.clear_ordering(True)
     882        super(SQLUpdateCompiler, self).pre_sql_setup()
     883        count = self.query.count_active_tables()
     884        if not self.query.related_updates and count == 1:
     885            return
     886
     887        # We need to use a sub-select in the where clause to filter on things
     888        # from other tables.
     889        query = self.query.clone(klass=Query)
     890        query.bump_prefix()
     891        query.extra = {}
     892        query.select = []
     893        query.add_fields([query.model._meta.pk.name])
     894        must_pre_select = count > 1 and not self.connection.features.update_can_self_select
     895
     896        # Now we adjust the current query: reset the where clause and get rid
     897        # of all the tables we don't need (since they're in the sub-select).
     898        self.query.where = self.query.where_class()
     899        if self.query.related_updates or must_pre_select:
     900            # Either we're using the idents in multiple update queries (so
     901            # don't want them to change), or the db backend doesn't support
     902            # selecting from the updating table (e.g. MySQL).
     903            idents = []
     904            for rows in query.get_compiler(self.using).execute_sql(MULTI):
     905                idents.extend([r[0] for r in rows])
     906            self.query.add_filter(('pk__in', idents))
     907            self.query.related_ids = idents
     908        else:
     909            # The fast path. Filters and updates in one query.
     910            self.query.add_filter(('pk__in', query))
     911        for alias in self.query.tables[1:]:
     912            self.query.alias_refcount[alias] = 0
     913
     914class SQLAggregateCompiler(SQLCompiler):
     915    def as_sql(self, qn=None):
     916        """
     917        Creates the SQL for this query. Returns the SQL string and list of
     918        parameters.
     919        """
     920        if qn is None:
     921            qn = self.quote_name_unless_alias
     922        sql = ('SELECT %s FROM (%s) subquery' % (
     923            ', '.join([
     924                aggregate.as_sql(qn, self.connection)
     925                for aggregate in self.query.aggregate_select.values()
     926            ]),
     927            self.query.subquery)
     928        )
     929        params = self.query.sub_params
     930        return (sql, params)
     931
     932class SQLDateCompiler(SQLCompiler):
     933    def results_iter(self):
     934        """
     935        Returns an iterator over the results from executing this query.
     936        """
     937        resolve_columns = hasattr(self, 'resolve_columns')
     938        if resolve_columns:
     939            from django.db.models.fields import DateTimeField
     940            fields = [DateTimeField()]
     941        else:
     942            from django.db.backends.util import typecast_timestamp
     943            needs_string_cast = self.connection.features.needs_datetime_string_cast
     944
     945        offset = len(self.query.extra_select)
     946        for rows in self.execute_sql(MULTI):
     947            for row in rows:
     948                date = row[offset]
     949                if resolve_columns:
     950                    date = self.resolve_columns(row, fields)[offset]
     951                elif needs_string_cast:
     952                    date = typecast_timestamp(str(date))
     953                yield date
     954
     955
     956def empty_iter():
     957    """
     958    Returns an iterator containing no results.
     959    """
     960    yield iter([]).next()
     961
     962
     963def order_modified_iter(cursor, trim, sentinel):
     964    """
     965    Yields blocks of rows from a cursor. We use this iterator in the special
     966    case when extra output columns have been added to support ordering
     967    requirements. We must trim those extra columns before anything else can use
     968    the results, since they're only needed to make the SQL valid.
     969    """
     970    for rows in iter((lambda: cursor.fetchmany(GET_ITERATOR_CHUNK_SIZE)),
     971            sentinel):
     972        yield [r[:-trim] for r in rows]
  • django/db/models/sql/query.py

    diff -rupN ../django-trunk/django/db/models/sql/query.py django/db/models/sql/query.py
    old new class Query(object): 
    130130
    131131        self.extra_tables = ()
    132132        self.extra_order_by = ()
     133        self.extra_join = ()
    133134
    134135        # A tuple that is a set of model field names and either True, if these
    135136        # are the fields to defer, or False if these are the only fields to
    class Query(object): 
    263264            obj._extra_select_cache = self._extra_select_cache.copy()
    264265        obj.extra_tables = self.extra_tables
    265266        obj.extra_order_by = self.extra_order_by
     267        obj.extra_join = self.extra_join
    266268        obj.deferred_loading = deepcopy(self.deferred_loading, memo=memo)
    267269        if self.filter_is_sticky and self.used_aliases:
    268270            obj.used_aliases = self.used_aliases.copy()
    class Query(object): 
    383385
    384386    def has_results(self, using):
    385387        q = self.clone()
    386         q.add_extra({'a': 1}, None, None, None, None, None)
     388        q.add_extra({'a': 1}, None, None, None, None, None, None)
    387389        q.select = []
    388390        q.select_fields = []
    389391        q.default_cols = False
    class Query(object): 
    15881590        self.related_select_cols = []
    15891591        self.related_select_fields = []
    15901592
    1591     def add_extra(self, select, select_params, where, params, tables, order_by):
     1593    def add_extra(self, select, select_params, where, params, tables, order_by, join):
    15921594        """
    15931595        Adds data to the various extra_* attributes for user-created additions
    15941596        to the query.
    class Query(object): 
    16191621            self.extra_tables += tuple(tables)
    16201622        if order_by:
    16211623            self.extra_order_by = order_by
     1624        if join:
     1625            self.extra_join += tuple(join)
    16221626
    16231627    def clear_deferred_loading(self):
    16241628        """
  • django/db/models/sql/query.py.orig

    diff -rupN ../django-trunk/django/db/models/sql/query.py.orig django/db/models/sql/query.py.orig
    old new  
     1"""
     2Create SQL statements for QuerySets.
     3
     4The code in here encapsulates all of the SQL construction so that QuerySets
     5themselves do not have to (and could be backed by things other than SQL
     6databases). The abstraction barrier only works one way: this module has to know
     7all about the internals of models in order to get the information it needs.
     8"""
     9
     10from django.utils.copycompat import deepcopy
     11from django.utils.tree import Node
     12from django.utils.datastructures import SortedDict
     13from django.utils.encoding import force_unicode
     14from django.db import connections, DEFAULT_DB_ALIAS
     15from django.db.models import signals
     16from django.db.models.fields import FieldDoesNotExist
     17from django.db.models.query_utils import select_related_descend, InvalidQuery
     18from django.db.models.sql import aggregates as base_aggregates_module
     19from django.db.models.sql.constants import *
     20from django.db.models.sql.datastructures import EmptyResultSet, Empty, MultiJoin
     21from django.db.models.sql.expressions import SQLEvaluator
     22from django.db.models.sql.where import (WhereNode, Constraint, EverythingNode,
     23    ExtraWhere, AND, OR)
     24from django.core.exceptions import FieldError
     25
     26__all__ = ['Query', 'RawQuery']
     27
     28class RawQuery(object):
     29    """
     30    A single raw SQL query
     31    """
     32
     33    def __init__(self, sql, using, params=None):
     34        self.validate_sql(sql)
     35        self.params = params or ()
     36        self.sql = sql
     37        self.using = using
     38        self.cursor = None
     39
     40    def clone(self, using):
     41        return RawQuery(self.sql, using, params=self.params)
     42
     43    def get_columns(self):
     44        if self.cursor is None:
     45            self._execute_query()
     46        converter = connections[self.using].introspection.table_name_converter
     47        return [converter(column_meta[0])
     48                for column_meta in self.cursor.description]
     49
     50    def validate_sql(self, sql):
     51        if not sql.lower().strip().startswith('select'):
     52            raise InvalidQuery('Raw queries are limited to SELECT queries. Use '
     53                               'connection.cursor directly for other types of queries.')
     54
     55    def __iter__(self):
     56        # Always execute a new query for a new iterator.
     57        # This could be optimized with a cache at the expense of RAM.
     58        self._execute_query()
     59        return iter(self.cursor)
     60
     61    def __repr__(self):
     62        return "<RawQuery: %r>" % (self.sql % self.params)
     63
     64    def _execute_query(self):
     65        self.cursor = connections[self.using].cursor()
     66        self.cursor.execute(self.sql, self.params)
     67
     68
     69class Query(object):
     70    """
     71    A single SQL query.
     72    """
     73    # SQL join types. These are part of the class because their string forms
     74    # vary from database to database and can be customised by a subclass.
     75    INNER = 'INNER JOIN'
     76    LOUTER = 'LEFT OUTER JOIN'
     77
     78    alias_prefix = 'T'
     79    query_terms = QUERY_TERMS
     80    aggregates_module = base_aggregates_module
     81
     82    compiler = 'SQLCompiler'
     83
     84    def __init__(self, model, where=WhereNode):
     85        self.model = model
     86        self.alias_refcount = {}
     87        self.alias_map = {}     # Maps alias to join information
     88        self.table_map = {}     # Maps table names to list of aliases.
     89        self.join_map = {}
     90        self.rev_join_map = {}  # Reverse of join_map.
     91        self.quote_cache = {}
     92        self.default_cols = True
     93        self.default_ordering = True
     94        self.standard_ordering = True
     95        self.ordering_aliases = []
     96        self.select_fields = []
     97        self.related_select_fields = []
     98        self.dupe_avoidance = {}
     99        self.used_aliases = set()
     100        self.filter_is_sticky = False
     101        self.included_inherited_models = {}
     102
     103        # SQL-related attributes
     104        self.select = []
     105        self.tables = []    # Aliases in the order they are created.
     106        self.where = where()
     107        self.where_class = where
     108        self.group_by = None
     109        self.having = where()
     110        self.order_by = []
     111        self.low_mark, self.high_mark = 0, None  # Used for offset/limit
     112        self.distinct = False
     113        self.select_related = False
     114        self.related_select_cols = []
     115
     116        # SQL aggregate-related attributes
     117        self.aggregates = SortedDict() # Maps alias -> SQL aggregate function
     118        self.aggregate_select_mask = None
     119        self._aggregate_select_cache = None
     120
     121        # Arbitrary maximum limit for select_related. Prevents infinite
     122        # recursion. Can be changed by the depth parameter to select_related().
     123        self.max_depth = 5
     124
     125        # These are for extensions. The contents are more or less appended
     126        # verbatim to the appropriate clause.
     127        self.extra = SortedDict()  # Maps col_alias -> (col_sql, params).
     128        self.extra_select_mask = None
     129        self._extra_select_cache = None
     130
     131        self.extra_tables = ()
     132        self.extra_order_by = ()
     133
     134        # A tuple that is a set of model field names and either True, if these
     135        # are the fields to defer, or False if these are the only fields to
     136        # load.
     137        self.deferred_loading = (set(), True)
     138
     139    def __str__(self):
     140        """
     141        Returns the query as a string of SQL with the parameter values
     142        substituted in.
     143
     144        Parameter values won't necessarily be quoted correctly, since that is
     145        done by the database interface at execution time.
     146        """
     147        sql, params = self.get_compiler(DEFAULT_DB_ALIAS).as_sql()
     148        return sql % params
     149
     150    def __deepcopy__(self, memo):
     151        result = self.clone(memo=memo)
     152        memo[id(self)] = result
     153        return result
     154
     155    def __getstate__(self):
     156        """
     157        Pickling support.
     158        """
     159        obj_dict = self.__dict__.copy()
     160        obj_dict['related_select_fields'] = []
     161        obj_dict['related_select_cols'] = []
     162
     163        # Fields can't be pickled, so if a field list has been
     164        # specified, we pickle the list of field names instead.
     165        # None is also a possible value; that can pass as-is
     166        obj_dict['select_fields'] = [
     167            f is not None and f.name or None
     168            for f in obj_dict['select_fields']
     169        ]
     170        return obj_dict
     171
     172    def __setstate__(self, obj_dict):
     173        """
     174        Unpickling support.
     175        """
     176        # Rebuild list of field instances
     177        obj_dict['select_fields'] = [
     178            name is not None and obj_dict['model']._meta.get_field(name) or None
     179            for name in obj_dict['select_fields']
     180        ]
     181
     182        self.__dict__.update(obj_dict)
     183
     184    def prepare(self):
     185        return self
     186
     187    def get_compiler(self, using=None, connection=None):
     188        if using is None and connection is None:
     189            raise ValueError("Need either using or connection")
     190        if using:
     191            connection = connections[using]
     192
     193        # Check that the compiler will be able to execute the query
     194        for alias, aggregate in self.aggregate_select.items():
     195            connection.ops.check_aggregate_support(aggregate)
     196
     197        return connection.ops.compiler(self.compiler)(self, connection, using)
     198
     199    def get_meta(self):
     200        """
     201        Returns the Options instance (the model._meta) from which to start
     202        processing. Normally, this is self.model._meta, but it can be changed
     203        by subclasses.
     204        """
     205        return self.model._meta
     206
     207    def clone(self, klass=None, memo=None, **kwargs):
     208        """
     209        Creates a copy of the current instance. The 'kwargs' parameter can be
     210        used by clients to update attributes after copying has taken place.
     211        """
     212        obj = Empty()
     213        obj.__class__ = klass or self.__class__
     214        obj.model = self.model
     215        obj.alias_refcount = self.alias_refcount.copy()
     216        obj.alias_map = self.alias_map.copy()
     217        obj.table_map = self.table_map.copy()
     218        obj.join_map = self.join_map.copy()
     219        obj.rev_join_map = self.rev_join_map.copy()
     220        obj.quote_cache = {}
     221        obj.default_cols = self.default_cols
     222        obj.default_ordering = self.default_ordering
     223        obj.standard_ordering = self.standard_ordering
     224        obj.included_inherited_models = self.included_inherited_models.copy()
     225        obj.ordering_aliases = []
     226        obj.select_fields = self.select_fields[:]
     227        obj.related_select_fields = self.related_select_fields[:]
     228        obj.dupe_avoidance = self.dupe_avoidance.copy()
     229        obj.select = self.select[:]
     230        obj.tables = self.tables[:]
     231        obj.where = deepcopy(self.where, memo=memo)
     232        obj.where_class = self.where_class
     233        if self.group_by is None:
     234            obj.group_by = None
     235        else:
     236            obj.group_by = self.group_by[:]
     237        obj.having = deepcopy(self.having, memo=memo)
     238        obj.order_by = self.order_by[:]
     239        obj.low_mark, obj.high_mark = self.low_mark, self.high_mark
     240        obj.distinct = self.distinct
     241        obj.select_related = self.select_related
     242        obj.related_select_cols = []
     243        obj.aggregates = deepcopy(self.aggregates, memo=memo)
     244        if self.aggregate_select_mask is None:
     245            obj.aggregate_select_mask = None
     246        else:
     247            obj.aggregate_select_mask = self.aggregate_select_mask.copy()
     248        # _aggregate_select_cache cannot be copied, as doing so breaks the
     249        # (necessary) state in which both aggregates and
     250        # _aggregate_select_cache point to the same underlying objects.
     251        # It will get re-populated in the cloned queryset the next time it's
     252        # used.
     253        obj._aggregate_select_cache = None
     254        obj.max_depth = self.max_depth
     255        obj.extra = self.extra.copy()
     256        if self.extra_select_mask is None:
     257            obj.extra_select_mask = None
     258        else:
     259            obj.extra_select_mask = self.extra_select_mask.copy()
     260        if self._extra_select_cache is None:
     261            obj._extra_select_cache = None
     262        else:
     263            obj._extra_select_cache = self._extra_select_cache.copy()
     264        obj.extra_tables = self.extra_tables
     265        obj.extra_order_by = self.extra_order_by
     266        obj.deferred_loading = deepcopy(self.deferred_loading, memo=memo)
     267        if self.filter_is_sticky and self.used_aliases:
     268            obj.used_aliases = self.used_aliases.copy()
     269        else:
     270            obj.used_aliases = set()
     271        obj.filter_is_sticky = False
     272        obj.__dict__.update(kwargs)
     273        if hasattr(obj, '_setup_query'):
     274            obj._setup_query()
     275        return obj
     276
     277    def convert_values(self, value, field, connection):
     278        """Convert the database-returned value into a type that is consistent
     279        across database backends.
     280
     281        By default, this defers to the underlying backend operations, but
     282        it can be overridden by Query classes for specific backends.
     283        """
     284        return connection.ops.convert_values(value, field)
     285
     286    def resolve_aggregate(self, value, aggregate, connection):
     287        """Resolve the value of aggregates returned by the database to
     288        consistent (and reasonable) types.
     289
     290        This is required because of the predisposition of certain backends
     291        to return Decimal and long types when they are not needed.
     292        """
     293        if value is None:
     294            if aggregate.is_ordinal:
     295                return 0
     296            # Return None as-is
     297            return value
     298        elif aggregate.is_ordinal:
     299            # Any ordinal aggregate (e.g., count) returns an int
     300            return int(value)
     301        elif aggregate.is_computed:
     302            # Any computed aggregate (e.g., avg) returns a float
     303            return float(value)
     304        else:
     305            # Return value depends on the type of the field being processed.
     306            return self.convert_values(value, aggregate.field, connection)
     307
     308    def get_aggregation(self, using):
     309        """
     310        Returns the dictionary with the values of the existing aggregations.
     311        """
     312        if not self.aggregate_select:
     313            return {}
     314
     315        # If there is a group by clause, aggregating does not add useful
     316        # information but retrieves only the first row. Aggregate
     317        # over the subquery instead.
     318        if self.group_by is not None:
     319            from subqueries import AggregateQuery
     320            query = AggregateQuery(self.model)
     321
     322            obj = self.clone()
     323
     324            # Remove any aggregates marked for reduction from the subquery
     325            # and move them to the outer AggregateQuery.
     326            for alias, aggregate in self.aggregate_select.items():
     327                if aggregate.is_summary:
     328                    query.aggregate_select[alias] = aggregate
     329                    del obj.aggregate_select[alias]
     330
     331            query.add_subquery(obj, using)
     332        else:
     333            query = self
     334            self.select = []
     335            self.default_cols = False
     336            self.extra = {}
     337            self.remove_inherited_models()
     338
     339        query.clear_ordering(True)
     340        query.clear_limits()
     341        query.select_related = False
     342        query.related_select_cols = []
     343        query.related_select_fields = []
     344
     345        result = query.get_compiler(using).execute_sql(SINGLE)
     346        if result is None:
     347            result = [None for q in query.aggregate_select.items()]
     348
     349        return dict([
     350            (alias, self.resolve_aggregate(val, aggregate, connection=connections[using]))
     351            for (alias, aggregate), val
     352            in zip(query.aggregate_select.items(), result)
     353        ])
     354
     355    def get_count(self, using):
     356        """
     357        Performs a COUNT() query using the current filter constraints.
     358        """
     359        obj = self.clone()
     360        if len(self.select) > 1 or self.aggregate_select:
     361            # If a select clause exists, then the query has already started to
     362            # specify the columns that are to be returned.
     363            # In this case, we need to use a subquery to evaluate the count.
     364            from subqueries import AggregateQuery
     365            subquery = obj
     366            subquery.clear_ordering(True)
     367            subquery.clear_limits()
     368
     369            obj = AggregateQuery(obj.model)
     370            obj.add_subquery(subquery, using=using)
     371
     372        obj.add_count_column()
     373        number = obj.get_aggregation(using=using)[None]
     374
     375        # Apply offset and limit constraints manually, since using LIMIT/OFFSET
     376        # in SQL (in variants that provide them) doesn't change the COUNT
     377        # output.
     378        number = max(0, number - self.low_mark)
     379        if self.high_mark is not None:
     380            number = min(number, self.high_mark - self.low_mark)
     381
     382        return number
     383
     384    def has_results(self, using):
     385        q = self.clone()
     386        q.add_extra({'a': 1}, None, None, None, None, None)
     387        q.select = []
     388        q.select_fields = []
     389        q.default_cols = False
     390        q.select_related = False
     391        q.set_extra_mask(('a',))
     392        q.set_aggregate_mask(())
     393        q.clear_ordering(True)
     394        q.set_limits(high=1)
     395        compiler = q.get_compiler(using=using)
     396        return bool(compiler.execute_sql(SINGLE))
     397
     398    def combine(self, rhs, connector):
     399        """
     400        Merge the 'rhs' query into the current one (with any 'rhs' effects
     401        being applied *after* (that is, "to the right of") anything in the
     402        current query. 'rhs' is not modified during a call to this function.
     403
     404        The 'connector' parameter describes how to connect filters from the
     405        'rhs' query.
     406        """
     407        assert self.model == rhs.model, \
     408                "Cannot combine queries on two different base models."
     409        assert self.can_filter(), \
     410                "Cannot combine queries once a slice has been taken."
     411        assert self.distinct == rhs.distinct, \
     412            "Cannot combine a unique query with a non-unique query."
     413
     414        self.remove_inherited_models()
     415        # Work out how to relabel the rhs aliases, if necessary.
     416        change_map = {}
     417        used = set()
     418        conjunction = (connector == AND)
     419        first = True
     420        for alias in rhs.tables:
     421            if not rhs.alias_refcount[alias]:
     422                # An unused alias.
     423                continue
     424            promote = (rhs.alias_map[alias][JOIN_TYPE] == self.LOUTER)
     425            new_alias = self.join(rhs.rev_join_map[alias],
     426                    (conjunction and not first), used, promote, not conjunction)
     427            used.add(new_alias)
     428            change_map[alias] = new_alias
     429            first = False
     430
     431        # So that we don't exclude valid results in an "or" query combination,
     432        # the first join that is exclusive to the lhs (self) must be converted
     433        # to an outer join.
     434        if not conjunction:
     435            for alias in self.tables[1:]:
     436                if self.alias_refcount[alias] == 1:
     437                    self.promote_alias(alias, True)
     438                    break
     439
     440        # Now relabel a copy of the rhs where-clause and add it to the current
     441        # one.
     442        if rhs.where:
     443            w = deepcopy(rhs.where)
     444            w.relabel_aliases(change_map)
     445            if not self.where:
     446                # Since 'self' matches everything, add an explicit "include
     447                # everything" where-constraint so that connections between the
     448                # where clauses won't exclude valid results.
     449                self.where.add(EverythingNode(), AND)
     450        elif self.where:
     451            # rhs has an empty where clause.
     452            w = self.where_class()
     453            w.add(EverythingNode(), AND)
     454        else:
     455            w = self.where_class()
     456        self.where.add(w, connector)
     457
     458        # Selection columns and extra extensions are those provided by 'rhs'.
     459        self.select = []
     460        for col in rhs.select:
     461            if isinstance(col, (list, tuple)):
     462                self.select.append((change_map.get(col[0], col[0]), col[1]))
     463            else:
     464                item = deepcopy(col)
     465                item.relabel_aliases(change_map)
     466                self.select.append(item)
     467        self.select_fields = rhs.select_fields[:]
     468
     469        if connector == OR:
     470            # It would be nice to be able to handle this, but the queries don't
     471            # really make sense (or return consistent value sets). Not worth
     472            # the extra complexity when you can write a real query instead.
     473            if self.extra and rhs.extra:
     474                raise ValueError("When merging querysets using 'or', you "
     475                        "cannot have extra(select=...) on both sides.")
     476        self.extra.update(rhs.extra)
     477        extra_select_mask = set()
     478        if self.extra_select_mask is not None:
     479            extra_select_mask.update(self.extra_select_mask)
     480        if rhs.extra_select_mask is not None:
     481            extra_select_mask.update(rhs.extra_select_mask)
     482        if extra_select_mask:
     483            self.set_extra_mask(extra_select_mask)
     484        self.extra_tables += rhs.extra_tables
     485
     486        # Ordering uses the 'rhs' ordering, unless it has none, in which case
     487        # the current ordering is used.
     488        self.order_by = rhs.order_by and rhs.order_by[:] or self.order_by
     489        self.extra_order_by = rhs.extra_order_by or self.extra_order_by
     490
     491    def deferred_to_data(self, target, callback):
     492        """
     493        Converts the self.deferred_loading data structure to an alternate data
     494        structure, describing the field that *will* be loaded. This is used to
     495        compute the columns to select from the database and also by the
     496        QuerySet class to work out which fields are being initialised on each
     497        model. Models that have all their fields included aren't mentioned in
     498        the result, only those that have field restrictions in place.
     499
     500        The "target" parameter is the instance that is populated (in place).
     501        The "callback" is a function that is called whenever a (model, field)
     502        pair need to be added to "target". It accepts three parameters:
     503        "target", and the model and list of fields being added for that model.
     504        """
     505        field_names, defer = self.deferred_loading
     506        if not field_names:
     507            return
     508        columns = set()
     509        orig_opts = self.model._meta
     510        seen = {}
     511        must_include = {self.model: set([orig_opts.pk])}
     512        for field_name in field_names:
     513            parts = field_name.split(LOOKUP_SEP)
     514            cur_model = self.model
     515            opts = orig_opts
     516            for name in parts[:-1]:
     517                old_model = cur_model
     518                source = opts.get_field_by_name(name)[0]
     519                cur_model = opts.get_field_by_name(name)[0].rel.to
     520                opts = cur_model._meta
     521                # Even if we're "just passing through" this model, we must add
     522                # both the current model's pk and the related reference field
     523                # to the things we select.
     524                must_include[old_model].add(source)
     525                add_to_dict(must_include, cur_model, opts.pk)
     526            field, model, _, _ = opts.get_field_by_name(parts[-1])
     527            if model is None:
     528                model = cur_model
     529            add_to_dict(seen, model, field)
     530
     531        if defer:
     532            # We need to load all fields for each model, except those that
     533            # appear in "seen" (for all models that appear in "seen"). The only
     534            # slight complexity here is handling fields that exist on parent
     535            # models.
     536            workset = {}
     537            for model, values in seen.iteritems():
     538                for field in model._meta.local_fields:
     539                    if field in values:
     540                        continue
     541                    add_to_dict(workset, model, field)
     542            for model, values in must_include.iteritems():
     543                # If we haven't included a model in workset, we don't add the
     544                # corresponding must_include fields for that model, since an
     545                # empty set means "include all fields". That's why there's no
     546                # "else" branch here.
     547                if model in workset:
     548                    workset[model].update(values)
     549            for model, values in workset.iteritems():
     550                callback(target, model, values)
     551        else:
     552            for model, values in must_include.iteritems():
     553                if model in seen:
     554                    seen[model].update(values)
     555                else:
     556                    # As we've passed through this model, but not explicitly
     557                    # included any fields, we have to make sure it's mentioned
     558                    # so that only the "must include" fields are pulled in.
     559                    seen[model] = values
     560            # Now ensure that every model in the inheritance chain is mentioned
     561            # in the parent list. Again, it must be mentioned to ensure that
     562            # only "must include" fields are pulled in.
     563            for model in orig_opts.get_parent_list():
     564                if model not in seen:
     565                    seen[model] = set()
     566            for model, values in seen.iteritems():
     567                callback(target, model, values)
     568
     569
     570    def deferred_to_columns_cb(self, target, model, fields):
     571        """
     572        Callback used by deferred_to_columns(). The "target" parameter should
     573        be a set instance.
     574        """
     575        table = model._meta.db_table
     576        if table not in target:
     577            target[table] = set()
     578        for field in fields:
     579            target[table].add(field.column)
     580
     581
     582    def table_alias(self, table_name, create=False):
     583        """
     584        Returns a table alias for the given table_name and whether this is a
     585        new alias or not.
     586
     587        If 'create' is true, a new alias is always created. Otherwise, the
     588        most recently created alias for the table (if one exists) is reused.
     589        """
     590        current = self.table_map.get(table_name)
     591        if not create and current:
     592            alias = current[0]
     593            self.alias_refcount[alias] += 1
     594            return alias, False
     595
     596        # Create a new alias for this table.
     597        if current:
     598            alias = '%s%d' % (self.alias_prefix, len(self.alias_map) + 1)
     599            current.append(alias)
     600        else:
     601            # The first occurence of a table uses the table name directly.
     602            alias = table_name
     603            self.table_map[alias] = [alias]
     604        self.alias_refcount[alias] = 1
     605        self.tables.append(alias)
     606        return alias, True
     607
     608    def ref_alias(self, alias):
     609        """ Increases the reference count for this alias. """
     610        self.alias_refcount[alias] += 1
     611
     612    def unref_alias(self, alias):
     613        """ Decreases the reference count for this alias. """
     614        self.alias_refcount[alias] -= 1
     615
     616    def promote_alias(self, alias, unconditional=False):
     617        """
     618        Promotes the join type of an alias to an outer join if it's possible
     619        for the join to contain NULL values on the left. If 'unconditional' is
     620        False, the join is only promoted if it is nullable, otherwise it is
     621        always promoted.
     622
     623        Returns True if the join was promoted.
     624        """
     625        if ((unconditional or self.alias_map[alias][NULLABLE]) and
     626                self.alias_map[alias][JOIN_TYPE] != self.LOUTER):
     627            data = list(self.alias_map[alias])
     628            data[JOIN_TYPE] = self.LOUTER
     629            self.alias_map[alias] = tuple(data)
     630            return True
     631        return False
     632
     633    def promote_alias_chain(self, chain, must_promote=False):
     634        """
     635        Walks along a chain of aliases, promoting the first nullable join and
     636        any joins following that. If 'must_promote' is True, all the aliases in
     637        the chain are promoted.
     638        """
     639        for alias in chain:
     640            if self.promote_alias(alias, must_promote):
     641                must_promote = True
     642
     643    def promote_unused_aliases(self, initial_refcounts, used_aliases):
     644        """
     645        Given a "before" copy of the alias_refcounts dictionary (as
     646        'initial_refcounts') and a collection of aliases that may have been
     647        changed or created, works out which aliases have been created since
     648        then and which ones haven't been used and promotes all of those
     649        aliases, plus any children of theirs in the alias tree, to outer joins.
     650        """
     651        # FIXME: There's some (a lot of!) overlap with the similar OR promotion
     652        # in add_filter(). It's not quite identical, but is very similar. So
     653        # pulling out the common bits is something for later.
     654        considered = {}
     655        for alias in self.tables:
     656            if alias not in used_aliases:
     657                continue
     658            if (alias not in initial_refcounts or
     659                    self.alias_refcount[alias] == initial_refcounts[alias]):
     660                parent = self.alias_map[alias][LHS_ALIAS]
     661                must_promote = considered.get(parent, False)
     662                promoted = self.promote_alias(alias, must_promote)
     663                considered[alias] = must_promote or promoted
     664
     665    def change_aliases(self, change_map):
     666        """
     667        Changes the aliases in change_map (which maps old-alias -> new-alias),
     668        relabelling any references to them in select columns and the where
     669        clause.
     670        """
     671        assert set(change_map.keys()).intersection(set(change_map.values())) == set()
     672
     673        # 1. Update references in "select" (normal columns plus aliases),
     674        # "group by", "where" and "having".
     675        self.where.relabel_aliases(change_map)
     676        self.having.relabel_aliases(change_map)
     677        for columns in (self.select, self.aggregates.values(), self.group_by or []):
     678            for pos, col in enumerate(columns):
     679                if isinstance(col, (list, tuple)):
     680                    old_alias = col[0]
     681                    columns[pos] = (change_map.get(old_alias, old_alias), col[1])
     682                else:
     683                    col.relabel_aliases(change_map)
     684
     685        # 2. Rename the alias in the internal table/alias datastructures.
     686        for old_alias, new_alias in change_map.iteritems():
     687            alias_data = list(self.alias_map[old_alias])
     688            alias_data[RHS_ALIAS] = new_alias
     689
     690            t = self.rev_join_map[old_alias]
     691            data = list(self.join_map[t])
     692            data[data.index(old_alias)] = new_alias
     693            self.join_map[t] = tuple(data)
     694            self.rev_join_map[new_alias] = t
     695            del self.rev_join_map[old_alias]
     696            self.alias_refcount[new_alias] = self.alias_refcount[old_alias]
     697            del self.alias_refcount[old_alias]
     698            self.alias_map[new_alias] = tuple(alias_data)
     699            del self.alias_map[old_alias]
     700
     701            table_aliases = self.table_map[alias_data[TABLE_NAME]]
     702            for pos, alias in enumerate(table_aliases):
     703                if alias == old_alias:
     704                    table_aliases[pos] = new_alias
     705                    break
     706            for pos, alias in enumerate(self.tables):
     707                if alias == old_alias:
     708                    self.tables[pos] = new_alias
     709                    break
     710        for key, alias in self.included_inherited_models.items():
     711            if alias in change_map:
     712                self.included_inherited_models[key] = change_map[alias]
     713
     714        # 3. Update any joins that refer to the old alias.
     715        for alias, data in self.alias_map.iteritems():
     716            lhs = data[LHS_ALIAS]
     717            if lhs in change_map:
     718                data = list(data)
     719                data[LHS_ALIAS] = change_map[lhs]
     720                self.alias_map[alias] = tuple(data)
     721
     722    def bump_prefix(self, exceptions=()):
     723        """
     724        Changes the alias prefix to the next letter in the alphabet and
     725        relabels all the aliases. Even tables that previously had no alias will
     726        get an alias after this call (it's mostly used for nested queries and
     727        the outer query will already be using the non-aliased table name).
     728
     729        Subclasses who create their own prefix should override this method to
     730        produce a similar result (a new prefix and relabelled aliases).
     731
     732        The 'exceptions' parameter is a container that holds alias names which
     733        should not be changed.
     734        """
     735        current = ord(self.alias_prefix)
     736        assert current < ord('Z')
     737        prefix = chr(current + 1)
     738        self.alias_prefix = prefix
     739        change_map = {}
     740        for pos, alias in enumerate(self.tables):
     741            if alias in exceptions:
     742                continue
     743            new_alias = '%s%d' % (prefix, pos)
     744            change_map[alias] = new_alias
     745            self.tables[pos] = new_alias
     746        self.change_aliases(change_map)
     747
     748    def get_initial_alias(self):
     749        """
     750        Returns the first alias for this query, after increasing its reference
     751        count.
     752        """
     753        if self.tables:
     754            alias = self.tables[0]
     755            self.ref_alias(alias)
     756        else:
     757            alias = self.join((None, self.model._meta.db_table, None, None))
     758        return alias
     759
     760    def count_active_tables(self):
     761        """
     762        Returns the number of tables in this query with a non-zero reference
     763        count.
     764        """
     765        return len([1 for count in self.alias_refcount.itervalues() if count])
     766
     767    def join(self, connection, always_create=False, exclusions=(),
     768            promote=False, outer_if_first=False, nullable=False, reuse=None):
     769        """
     770        Returns an alias for the join in 'connection', either reusing an
     771        existing alias for that join or creating a new one. 'connection' is a
     772        tuple (lhs, table, lhs_col, col) where 'lhs' is either an existing
     773        table alias or a table name. The join correspods to the SQL equivalent
     774        of::
     775
     776            lhs.lhs_col = table.col
     777
     778        If 'always_create' is True and 'reuse' is None, a new alias is always
     779        created, regardless of whether one already exists or not. If
     780        'always_create' is True and 'reuse' is a set, an alias in 'reuse' that
     781        matches the connection will be returned, if possible.  If
     782        'always_create' is False, the first existing alias that matches the
     783        'connection' is returned, if any. Otherwise a new join is created.
     784
     785        If 'exclusions' is specified, it is something satisfying the container
     786        protocol ("foo in exclusions" must work) and specifies a list of
     787        aliases that should not be returned, even if they satisfy the join.
     788
     789        If 'promote' is True, the join type for the alias will be LOUTER (if
     790        the alias previously existed, the join type will be promoted from INNER
     791        to LOUTER, if necessary).
     792
     793        If 'outer_if_first' is True and a new join is created, it will have the
     794        LOUTER join type. This is used when joining certain types of querysets
     795        and Q-objects together.
     796
     797        If 'nullable' is True, the join can potentially involve NULL values and
     798        is a candidate for promotion (to "left outer") when combining querysets.
     799        """
     800        lhs, table, lhs_col, col = connection
     801        if lhs in self.alias_map:
     802            lhs_table = self.alias_map[lhs][TABLE_NAME]
     803        else:
     804            lhs_table = lhs
     805
     806        if reuse and always_create and table in self.table_map:
     807            # Convert the 'reuse' to case to be "exclude everything but the
     808            # reusable set, minus exclusions, for this table".
     809            exclusions = set(self.table_map[table]).difference(reuse).union(set(exclusions))
     810            always_create = False
     811        t_ident = (lhs_table, table, lhs_col, col)
     812        if not always_create:
     813            for alias in self.join_map.get(t_ident, ()):
     814                if alias not in exclusions:
     815                    if lhs_table and not self.alias_refcount[self.alias_map[alias][LHS_ALIAS]]:
     816                        # The LHS of this join tuple is no longer part of the
     817                        # query, so skip this possibility.
     818                        continue
     819                    if self.alias_map[alias][LHS_ALIAS] != lhs:
     820                        continue
     821                    self.ref_alias(alias)
     822                    if promote:
     823                        self.promote_alias(alias)
     824                    return alias
     825
     826        # No reuse is possible, so we need a new alias.
     827        alias, _ = self.table_alias(table, True)
     828        if not lhs:
     829            # Not all tables need to be joined to anything. No join type
     830            # means the later columns are ignored.
     831            join_type = None
     832        elif promote or outer_if_first:
     833            join_type = self.LOUTER
     834        else:
     835            join_type = self.INNER
     836        join = (table, alias, join_type, lhs, lhs_col, col, nullable)
     837        self.alias_map[alias] = join
     838        if t_ident in self.join_map:
     839            self.join_map[t_ident] += (alias,)
     840        else:
     841            self.join_map[t_ident] = (alias,)
     842        self.rev_join_map[alias] = t_ident
     843        return alias
     844
     845    def setup_inherited_models(self):
     846        """
     847        If the model that is the basis for this QuerySet inherits other models,
     848        we need to ensure that those other models have their tables included in
     849        the query.
     850
     851        We do this as a separate step so that subclasses know which
     852        tables are going to be active in the query, without needing to compute
     853        all the select columns (this method is called from pre_sql_setup(),
     854        whereas column determination is a later part, and side-effect, of
     855        as_sql()).
     856        """
     857        opts = self.model._meta
     858        root_alias = self.tables[0]
     859        seen = {None: root_alias}
     860
     861        # Skip all proxy to the root proxied model
     862        proxied_model = get_proxied_model(opts)
     863
     864        for field, model in opts.get_fields_with_model():
     865            if model not in seen:
     866                if model is proxied_model:
     867                    seen[model] = root_alias
     868                else:
     869                    link_field = opts.get_ancestor_link(model)
     870                    seen[model] = self.join((root_alias, model._meta.db_table,
     871                            link_field.column, model._meta.pk.column))
     872        self.included_inherited_models = seen
     873
     874    def remove_inherited_models(self):
     875        """
     876        Undoes the effects of setup_inherited_models(). Should be called
     877        whenever select columns (self.select) are set explicitly.
     878        """
     879        for key, alias in self.included_inherited_models.items():
     880            if key:
     881                self.unref_alias(alias)
     882        self.included_inherited_models = {}
     883
     884
     885    def add_aggregate(self, aggregate, model, alias, is_summary):
     886        """
     887        Adds a single aggregate expression to the Query
     888        """
     889        opts = model._meta
     890        field_list = aggregate.lookup.split(LOOKUP_SEP)
     891        if (len(field_list) == 1 and
     892            aggregate.lookup in self.aggregates.keys()):
     893            # Aggregate is over an annotation
     894            field_name = field_list[0]
     895            col = field_name
     896            source = self.aggregates[field_name]
     897            if not is_summary:
     898                raise FieldError("Cannot compute %s('%s'): '%s' is an aggregate" % (
     899                    aggregate.name, field_name, field_name))
     900        elif ((len(field_list) > 1) or
     901            (field_list[0] not in [i.name for i in opts.fields]) or
     902            self.group_by is None or
     903            not is_summary):
     904            # If:
     905            #   - the field descriptor has more than one part (foo__bar), or
     906            #   - the field descriptor is referencing an m2m/m2o field, or
     907            #   - this is a reference to a model field (possibly inherited), or
     908            #   - this is an annotation over a model field
     909            # then we need to explore the joins that are required.
     910
     911            field, source, opts, join_list, last, _ = self.setup_joins(
     912                field_list, opts, self.get_initial_alias(), False)
     913
     914            # Process the join chain to see if it can be trimmed
     915            col, _, join_list = self.trim_joins(source, join_list, last, False)
     916
     917            # If the aggregate references a model or field that requires a join,
     918            # those joins must be LEFT OUTER - empty join rows must be returned
     919            # in order for zeros to be returned for those aggregates.
     920            for column_alias in join_list:
     921                self.promote_alias(column_alias, unconditional=True)
     922
     923            col = (join_list[-1], col)
     924        else:
     925            # The simplest cases. No joins required -
     926            # just reference the provided column alias.
     927            field_name = field_list[0]
     928            source = opts.get_field(field_name)
     929            col = field_name
     930
     931        # Add the aggregate to the query
     932        aggregate.add_to_query(self, alias, col=col, source=source, is_summary=is_summary)
     933
     934    def add_filter(self, filter_expr, connector=AND, negate=False, trim=False,
     935            can_reuse=None, process_extras=True):
     936        """
     937        Add a single filter to the query. The 'filter_expr' is a pair:
     938        (filter_string, value). E.g. ('name__contains', 'fred')
     939
     940        If 'negate' is True, this is an exclude() filter. It's important to
     941        note that this method does not negate anything in the where-clause
     942        object when inserting the filter constraints. This is because negated
     943        filters often require multiple calls to add_filter() and the negation
     944        should only happen once. So the caller is responsible for this (the
     945        caller will normally be add_q(), so that as an example).
     946
     947        If 'trim' is True, we automatically trim the final join group (used
     948        internally when constructing nested queries).
     949
     950        If 'can_reuse' is a set, we are processing a component of a
     951        multi-component filter (e.g. filter(Q1, Q2)). In this case, 'can_reuse'
     952        will be a set of table aliases that can be reused in this filter, even
     953        if we would otherwise force the creation of new aliases for a join
     954        (needed for nested Q-filters). The set is updated by this method.
     955
     956        If 'process_extras' is set, any extra filters returned from the table
     957        joining process will be processed. This parameter is set to False
     958        during the processing of extra filters to avoid infinite recursion.
     959        """
     960        arg, value = filter_expr
     961        parts = arg.split(LOOKUP_SEP)
     962        if not parts:
     963            raise FieldError("Cannot parse keyword query %r" % arg)
     964
     965        # Work out the lookup type and remove it from 'parts', if necessary.
     966        if len(parts) == 1 or parts[-1] not in self.query_terms:
     967            lookup_type = 'exact'
     968        else:
     969            lookup_type = parts.pop()
     970
     971        # By default, this is a WHERE clause. If an aggregate is referenced
     972        # in the value, the filter will be promoted to a HAVING
     973        having_clause = False
     974
     975        # Interpret '__exact=None' as the sql 'is NULL'; otherwise, reject all
     976        # uses of None as a query value.
     977        if value is None:
     978            if lookup_type != 'exact':
     979                raise ValueError("Cannot use None as a query value")
     980            lookup_type = 'isnull'
     981            value = True
     982        elif callable(value):
     983            value = value()
     984        elif hasattr(value, 'evaluate'):
     985            # If value is a query expression, evaluate it
     986            value = SQLEvaluator(value, self)
     987            having_clause = value.contains_aggregate
     988
     989        for alias, aggregate in self.aggregates.items():
     990            if alias == parts[0]:
     991                entry = self.where_class()
     992                entry.add((aggregate, lookup_type, value), AND)
     993                if negate:
     994                    entry.negate()
     995                self.having.add(entry, AND)
     996                return
     997
     998        opts = self.get_meta()
     999        alias = self.get_initial_alias()
     1000        allow_many = trim or not negate
     1001
     1002        try:
     1003            field, target, opts, join_list, last, extra_filters = self.setup_joins(
     1004                    parts, opts, alias, True, allow_many, can_reuse=can_reuse,
     1005                    negate=negate, process_extras=process_extras)
     1006        except MultiJoin, e:
     1007            self.split_exclude(filter_expr, LOOKUP_SEP.join(parts[:e.level]),
     1008                    can_reuse)
     1009            return
     1010
     1011        if (lookup_type == 'isnull' and value is True and not negate and
     1012                len(join_list) > 1):
     1013            # If the comparison is against NULL, we may need to use some left
     1014            # outer joins when creating the join chain. This is only done when
     1015            # needed, as it's less efficient at the database level.
     1016            self.promote_alias_chain(join_list)
     1017
     1018        # Process the join list to see if we can remove any inner joins from
     1019        # the far end (fewer tables in a query is better).
     1020        col, alias, join_list = self.trim_joins(target, join_list, last, trim)
     1021
     1022        if connector == OR:
     1023            # Some joins may need to be promoted when adding a new filter to a
     1024            # disjunction. We walk the list of new joins and where it diverges
     1025            # from any previous joins (ref count is 1 in the table list), we
     1026            # make the new additions (and any existing ones not used in the new
     1027            # join list) an outer join.
     1028            join_it = iter(join_list)
     1029            table_it = iter(self.tables)
     1030            join_it.next(), table_it.next()
     1031            table_promote = False
     1032            join_promote = False
     1033            for join in join_it:
     1034                table = table_it.next()
     1035                if join == table and self.alias_refcount[join] > 1:
     1036                    continue
     1037                join_promote = self.promote_alias(join)
     1038                if table != join:
     1039                    table_promote = self.promote_alias(table)
     1040                break
     1041            self.promote_alias_chain(join_it, join_promote)
     1042            self.promote_alias_chain(table_it, table_promote)
     1043
     1044
     1045        if having_clause:
     1046            self.having.add((Constraint(alias, col, field), lookup_type, value),
     1047                connector)
     1048        else:
     1049            self.where.add((Constraint(alias, col, field), lookup_type, value),
     1050                connector)
     1051
     1052        if negate:
     1053            self.promote_alias_chain(join_list)
     1054            if lookup_type != 'isnull':
     1055                if len(join_list) > 1:
     1056                    for alias in join_list:
     1057                        if self.alias_map[alias][JOIN_TYPE] == self.LOUTER:
     1058                            j_col = self.alias_map[alias][RHS_JOIN_COL]
     1059                            entry = self.where_class()
     1060                            entry.add((Constraint(alias, j_col, None), 'isnull', True), AND)
     1061                            entry.negate()
     1062                            self.where.add(entry, AND)
     1063                            break
     1064                elif not (lookup_type == 'in'
     1065                            and not hasattr(value, 'as_sql')
     1066                            and not hasattr(value, '_as_sql')
     1067                            and not value) and field.null:
     1068                    # Leaky abstraction artifact: We have to specifically
     1069                    # exclude the "foo__in=[]" case from this handling, because
     1070                    # it's short-circuited in the Where class.
     1071                    # We also need to handle the case where a subquery is provided
     1072                    entry = self.where_class()
     1073                    entry.add((Constraint(alias, col, None), 'isnull', True), AND)
     1074                    entry.negate()
     1075                    self.where.add(entry, AND)
     1076
     1077        if can_reuse is not None:
     1078            can_reuse.update(join_list)
     1079        if process_extras:
     1080            for filter in extra_filters:
     1081                self.add_filter(filter, negate=negate, can_reuse=can_reuse,
     1082                        process_extras=False)
     1083
     1084    def add_q(self, q_object, used_aliases=None):
     1085        """
     1086        Adds a Q-object to the current filter.
     1087
     1088        Can also be used to add anything that has an 'add_to_query()' method.
     1089        """
     1090        if used_aliases is None:
     1091            used_aliases = self.used_aliases
     1092        if hasattr(q_object, 'add_to_query'):
     1093            # Complex custom objects are responsible for adding themselves.
     1094            q_object.add_to_query(self, used_aliases)
     1095        else:
     1096            if self.where and q_object.connector != AND and len(q_object) > 1:
     1097                self.where.start_subtree(AND)
     1098                subtree = True
     1099            else:
     1100                subtree = False
     1101            connector = AND
     1102            for child in q_object.children:
     1103                if connector == OR:
     1104                    refcounts_before = self.alias_refcount.copy()
     1105                self.where.start_subtree(connector)
     1106                if isinstance(child, Node):
     1107                    self.add_q(child, used_aliases)
     1108                else:
     1109                    self.add_filter(child, connector, q_object.negated,
     1110                            can_reuse=used_aliases)
     1111                self.where.end_subtree()
     1112                if connector == OR:
     1113                    # Aliases that were newly added or not used at all need to
     1114                    # be promoted to outer joins if they are nullable relations.
     1115                    # (they shouldn't turn the whole conditional into the empty
     1116                    # set just because they don't match anything).
     1117                    self.promote_unused_aliases(refcounts_before, used_aliases)
     1118                connector = q_object.connector
     1119            if q_object.negated:
     1120                self.where.negate()
     1121            if subtree:
     1122                self.where.end_subtree()
     1123        if self.filter_is_sticky:
     1124            self.used_aliases = used_aliases
     1125
     1126    def setup_joins(self, names, opts, alias, dupe_multis, allow_many=True,
     1127            allow_explicit_fk=False, can_reuse=None, negate=False,
     1128            process_extras=True):
     1129        """
     1130        Compute the necessary table joins for the passage through the fields
     1131        given in 'names'. 'opts' is the Options class for the current model
     1132        (which gives the table we are joining to), 'alias' is the alias for the
     1133        table we are joining to. If dupe_multis is True, any many-to-many or
     1134        many-to-one joins will always create a new alias (necessary for
     1135        disjunctive filters). If can_reuse is not None, it's a list of aliases
     1136        that can be reused in these joins (nothing else can be reused in this
     1137        case). Finally, 'negate' is used in the same sense as for add_filter()
     1138        -- it indicates an exclude() filter, or something similar. It is only
     1139        passed in here so that it can be passed to a field's extra_filter() for
     1140        customised behaviour.
     1141
     1142        Returns the final field involved in the join, the target database
     1143        column (used for any 'where' constraint), the final 'opts' value and the
     1144        list of tables joined.
     1145        """
     1146        joins = [alias]
     1147        last = [0]
     1148        dupe_set = set()
     1149        exclusions = set()
     1150        extra_filters = []
     1151        for pos, name in enumerate(names):
     1152            try:
     1153                exclusions.add(int_alias)
     1154            except NameError:
     1155                pass
     1156            exclusions.add(alias)
     1157            last.append(len(joins))
     1158            if name == 'pk':
     1159                name = opts.pk.name
     1160            try:
     1161                field, model, direct, m2m = opts.get_field_by_name(name)
     1162            except FieldDoesNotExist:
     1163                for f in opts.fields:
     1164                    if allow_explicit_fk and name == f.attname:
     1165                        # XXX: A hack to allow foo_id to work in values() for
     1166                        # backwards compatibility purposes. If we dropped that
     1167                        # feature, this could be removed.
     1168                        field, model, direct, m2m = opts.get_field_by_name(f.name)
     1169                        break
     1170                else:
     1171                    names = opts.get_all_field_names() + self.aggregate_select.keys()
     1172                    raise FieldError("Cannot resolve keyword %r into field. "
     1173                            "Choices are: %s" % (name, ", ".join(names)))
     1174
     1175            if not allow_many and (m2m or not direct):
     1176                for alias in joins:
     1177                    self.unref_alias(alias)
     1178                raise MultiJoin(pos + 1)
     1179            if model:
     1180                # The field lives on a base class of the current model.
     1181                # Skip the chain of proxy to the concrete proxied model
     1182                proxied_model = get_proxied_model(opts)
     1183
     1184                for int_model in opts.get_base_chain(model):
     1185                    if int_model is proxied_model:
     1186                        opts = int_model._meta
     1187                    else:
     1188                        lhs_col = opts.parents[int_model].column
     1189                        dedupe = lhs_col in opts.duplicate_targets
     1190                        if dedupe:
     1191                            exclusions.update(self.dupe_avoidance.get(
     1192                                    (id(opts), lhs_col), ()))
     1193                            dupe_set.add((opts, lhs_col))
     1194                        opts = int_model._meta
     1195                        alias = self.join((alias, opts.db_table, lhs_col,
     1196                                opts.pk.column), exclusions=exclusions)
     1197                        joins.append(alias)
     1198                        exclusions.add(alias)
     1199                        for (dupe_opts, dupe_col) in dupe_set:
     1200                            self.update_dupe_avoidance(dupe_opts, dupe_col,
     1201                                    alias)
     1202            cached_data = opts._join_cache.get(name)
     1203            orig_opts = opts
     1204            dupe_col = direct and field.column or field.field.column
     1205            dedupe = dupe_col in opts.duplicate_targets
     1206            if dupe_set or dedupe:
     1207                if dedupe:
     1208                    dupe_set.add((opts, dupe_col))
     1209                exclusions.update(self.dupe_avoidance.get((id(opts), dupe_col),
     1210                        ()))
     1211
     1212            if process_extras and hasattr(field, 'extra_filters'):
     1213                extra_filters.extend(field.extra_filters(names, pos, negate))
     1214            if direct:
     1215                if m2m:
     1216                    # Many-to-many field defined on the current model.
     1217                    if cached_data:
     1218                        (table1, from_col1, to_col1, table2, from_col2,
     1219                                to_col2, opts, target) = cached_data
     1220                    else:
     1221                        table1 = field.m2m_db_table()
     1222                        from_col1 = opts.pk.column
     1223                        to_col1 = field.m2m_column_name()
     1224                        opts = field.rel.to._meta
     1225                        table2 = opts.db_table
     1226                        from_col2 = field.m2m_reverse_name()
     1227                        to_col2 = opts.pk.column
     1228                        target = opts.pk
     1229                        orig_opts._join_cache[name] = (table1, from_col1,
     1230                                to_col1, table2, from_col2, to_col2, opts,
     1231                                target)
     1232
     1233                    int_alias = self.join((alias, table1, from_col1, to_col1),
     1234                            dupe_multis, exclusions, nullable=True,
     1235                            reuse=can_reuse)
     1236                    if int_alias == table2 and from_col2 == to_col2:
     1237                        joins.append(int_alias)
     1238                        alias = int_alias
     1239                    else:
     1240                        alias = self.join(
     1241                                (int_alias, table2, from_col2, to_col2),
     1242                                dupe_multis, exclusions, nullable=True,
     1243                                reuse=can_reuse)
     1244                        joins.extend([int_alias, alias])
     1245                elif field.rel:
     1246                    # One-to-one or many-to-one field
     1247                    if cached_data:
     1248                        (table, from_col, to_col, opts, target) = cached_data
     1249                    else:
     1250                        opts = field.rel.to._meta
     1251                        target = field.rel.get_related_field()
     1252                        table = opts.db_table
     1253                        from_col = field.column
     1254                        to_col = target.column
     1255                        orig_opts._join_cache[name] = (table, from_col, to_col,
     1256                                opts, target)
     1257
     1258                    alias = self.join((alias, table, from_col, to_col),
     1259                            exclusions=exclusions, nullable=field.null)
     1260                    joins.append(alias)
     1261                else:
     1262                    # Non-relation fields.
     1263                    target = field
     1264                    break
     1265            else:
     1266                orig_field = field
     1267                field = field.field
     1268                if m2m:
     1269                    # Many-to-many field defined on the target model.
     1270                    if cached_data:
     1271                        (table1, from_col1, to_col1, table2, from_col2,
     1272                                to_col2, opts, target) = cached_data
     1273                    else:
     1274                        table1 = field.m2m_db_table()
     1275                        from_col1 = opts.pk.column
     1276                        to_col1 = field.m2m_reverse_name()
     1277                        opts = orig_field.opts
     1278                        table2 = opts.db_table
     1279                        from_col2 = field.m2m_column_name()
     1280                        to_col2 = opts.pk.column
     1281                        target = opts.pk
     1282                        orig_opts._join_cache[name] = (table1, from_col1,
     1283                                to_col1, table2, from_col2, to_col2, opts,
     1284                                target)
     1285
     1286                    int_alias = self.join((alias, table1, from_col1, to_col1),
     1287                            dupe_multis, exclusions, nullable=True,
     1288                            reuse=can_reuse)
     1289                    alias = self.join((int_alias, table2, from_col2, to_col2),
     1290                            dupe_multis, exclusions, nullable=True,
     1291                            reuse=can_reuse)
     1292                    joins.extend([int_alias, alias])
     1293                else:
     1294                    # One-to-many field (ForeignKey defined on the target model)
     1295                    if cached_data:
     1296                        (table, from_col, to_col, opts, target) = cached_data
     1297                    else:
     1298                        local_field = opts.get_field_by_name(
     1299                                field.rel.field_name)[0]
     1300                        opts = orig_field.opts
     1301                        table = opts.db_table
     1302                        from_col = local_field.column
     1303                        to_col = field.column
     1304                        target = opts.pk
     1305                        orig_opts._join_cache[name] = (table, from_col, to_col,
     1306                                opts, target)
     1307
     1308                    alias = self.join((alias, table, from_col, to_col),
     1309                            dupe_multis, exclusions, nullable=True,
     1310                            reuse=can_reuse)
     1311                    joins.append(alias)
     1312
     1313            for (dupe_opts, dupe_col) in dupe_set:
     1314                try:
     1315                    self.update_dupe_avoidance(dupe_opts, dupe_col, int_alias)
     1316                except NameError:
     1317                    self.update_dupe_avoidance(dupe_opts, dupe_col, alias)
     1318
     1319        if pos != len(names) - 1:
     1320            if pos == len(names) - 2:
     1321                raise FieldError("Join on field %r not permitted. Did you misspell %r for the lookup type?" % (name, names[pos + 1]))
     1322            else:
     1323                raise FieldError("Join on field %r not permitted." % name)
     1324
     1325        return field, target, opts, joins, last, extra_filters
     1326
     1327    def trim_joins(self, target, join_list, last, trim):
     1328        """
     1329        Sometimes joins at the end of a multi-table sequence can be trimmed. If
     1330        the final join is against the same column as we are comparing against,
     1331        and is an inner join, we can go back one step in a join chain and
     1332        compare against the LHS of the join instead (and then repeat the
     1333        optimization). The result, potentially, involves less table joins.
     1334
     1335        The 'target' parameter is the final field being joined to, 'join_list'
     1336        is the full list of join aliases.
     1337
     1338        The 'last' list contains offsets into 'join_list', corresponding to
     1339        each component of the filter.  Many-to-many relations, for example, add
     1340        two tables to the join list and we want to deal with both tables the
     1341        same way, so 'last' has an entry for the first of the two tables and
     1342        then the table immediately after the second table, in that case.
     1343
     1344        The 'trim' parameter forces the final piece of the join list to be
     1345        trimmed before anything. See the documentation of add_filter() for
     1346        details about this.
     1347
     1348        Returns the final active column and table alias and the new active
     1349        join_list.
     1350        """
     1351        final = len(join_list)
     1352        penultimate = last.pop()
     1353        if penultimate == final:
     1354            penultimate = last.pop()
     1355        if trim and len(join_list) > 1:
     1356            extra = join_list[penultimate:]
     1357            join_list = join_list[:penultimate]
     1358            final = penultimate
     1359            penultimate = last.pop()
     1360            col = self.alias_map[extra[0]][LHS_JOIN_COL]
     1361            for alias in extra:
     1362                self.unref_alias(alias)
     1363        else:
     1364            col = target.column
     1365        alias = join_list[-1]
     1366        while final > 1:
     1367            join = self.alias_map[alias]
     1368            if col != join[RHS_JOIN_COL] or join[JOIN_TYPE] != self.INNER:
     1369                break
     1370            self.unref_alias(alias)
     1371            alias = join[LHS_ALIAS]
     1372            col = join[LHS_JOIN_COL]
     1373            join_list = join_list[:-1]
     1374            final -= 1
     1375            if final == penultimate:
     1376                penultimate = last.pop()
     1377        return col, alias, join_list
     1378
     1379    def update_dupe_avoidance(self, opts, col, alias):
     1380        """
     1381        For a column that is one of multiple pointing to the same table, update
     1382        the internal data structures to note that this alias shouldn't be used
     1383        for those other columns.
     1384        """
     1385        ident = id(opts)
     1386        for name in opts.duplicate_targets[col]:
     1387            try:
     1388                self.dupe_avoidance[ident, name].add(alias)
     1389            except KeyError:
     1390                self.dupe_avoidance[ident, name] = set([alias])
     1391
     1392    def split_exclude(self, filter_expr, prefix, can_reuse):
     1393        """
     1394        When doing an exclude against any kind of N-to-many relation, we need
     1395        to use a subquery. This method constructs the nested query, given the
     1396        original exclude filter (filter_expr) and the portion up to the first
     1397        N-to-many relation field.
     1398        """
     1399        query = Query(self.model)
     1400        query.add_filter(filter_expr, can_reuse=can_reuse)
     1401        query.bump_prefix()
     1402        query.clear_ordering(True)
     1403        query.set_start(prefix)
     1404        self.add_filter(('%s__in' % prefix, query), negate=True, trim=True,
     1405                can_reuse=can_reuse)
     1406
     1407        # If there's more than one join in the inner query (before any initial
     1408        # bits were trimmed -- which means the last active table is more than
     1409        # two places into the alias list), we need to also handle the
     1410        # possibility that the earlier joins don't match anything by adding a
     1411        # comparison to NULL (e.g. in
     1412        # Tag.objects.exclude(parent__parent__name='t1'), a tag with no parent
     1413        # would otherwise be overlooked).
     1414        active_positions = [pos for (pos, count) in
     1415                enumerate(query.alias_refcount.itervalues()) if count]
     1416        if active_positions[-1] > 1:
     1417            self.add_filter(('%s__isnull' % prefix, False), negate=True,
     1418                    trim=True, can_reuse=can_reuse)
     1419
     1420    def set_limits(self, low=None, high=None):
     1421        """
     1422        Adjusts the limits on the rows retrieved. We use low/high to set these,
     1423        as it makes it more Pythonic to read and write. When the SQL query is
     1424        created, they are converted to the appropriate offset and limit values.
     1425
     1426        Any limits passed in here are applied relative to the existing
     1427        constraints. So low is added to the current low value and both will be
     1428        clamped to any existing high value.
     1429        """
     1430        if high is not None:
     1431            if self.high_mark is not None:
     1432                self.high_mark = min(self.high_mark, self.low_mark + high)
     1433            else:
     1434                self.high_mark = self.low_mark + high
     1435        if low is not None:
     1436            if self.high_mark is not None:
     1437                self.low_mark = min(self.high_mark, self.low_mark + low)
     1438            else:
     1439                self.low_mark = self.low_mark + low
     1440
     1441    def clear_limits(self):
     1442        """
     1443        Clears any existing limits.
     1444        """
     1445        self.low_mark, self.high_mark = 0, None
     1446
     1447    def can_filter(self):
     1448        """
     1449        Returns True if adding filters to this instance is still possible.
     1450
     1451        Typically, this means no limits or offsets have been put on the results.
     1452        """
     1453        return not self.low_mark and self.high_mark is None
     1454
     1455    def clear_select_fields(self):
     1456        """
     1457        Clears the list of fields to select (but not extra_select columns).
     1458        Some queryset types completely replace any existing list of select
     1459        columns.
     1460        """
     1461        self.select = []
     1462        self.select_fields = []
     1463
     1464    def add_fields(self, field_names, allow_m2m=True):
     1465        """
     1466        Adds the given (model) fields to the select set. The field names are
     1467        added in the order specified.
     1468        """
     1469        alias = self.get_initial_alias()
     1470        opts = self.get_meta()
     1471
     1472        try:
     1473            for name in field_names:
     1474                field, target, u2, joins, u3, u4 = self.setup_joins(
     1475                        name.split(LOOKUP_SEP), opts, alias, False, allow_m2m,
     1476                        True)
     1477                final_alias = joins[-1]
     1478                col = target.column
     1479                if len(joins) > 1:
     1480                    join = self.alias_map[final_alias]
     1481                    if col == join[RHS_JOIN_COL]:
     1482                        self.unref_alias(final_alias)
     1483                        final_alias = join[LHS_ALIAS]
     1484                        col = join[LHS_JOIN_COL]
     1485                        joins = joins[:-1]
     1486                self.promote_alias_chain(joins[1:])
     1487                self.select.append((final_alias, col))
     1488                self.select_fields.append(field)
     1489        except MultiJoin:
     1490            raise FieldError("Invalid field name: '%s'" % name)
     1491        except FieldError:
     1492            names = opts.get_all_field_names() + self.extra.keys() + self.aggregate_select.keys()
     1493            names.sort()
     1494            raise FieldError("Cannot resolve keyword %r into field. "
     1495                    "Choices are: %s" % (name, ", ".join(names)))
     1496        self.remove_inherited_models()
     1497
     1498    def add_ordering(self, *ordering):
     1499        """
     1500        Adds items from the 'ordering' sequence to the query's "order by"
     1501        clause. These items are either field names (not column names) --
     1502        possibly with a direction prefix ('-' or '?') -- or ordinals,
     1503        corresponding to column positions in the 'select' list.
     1504
     1505        If 'ordering' is empty, all ordering is cleared from the query.
     1506        """
     1507        errors = []
     1508        for item in ordering:
     1509            if not ORDER_PATTERN.match(item):
     1510                errors.append(item)
     1511        if errors:
     1512            raise FieldError('Invalid order_by arguments: %s' % errors)
     1513        if ordering:
     1514            self.order_by.extend(ordering)
     1515        else:
     1516            self.default_ordering = False
     1517
     1518    def clear_ordering(self, force_empty=False):
     1519        """
     1520        Removes any ordering settings. If 'force_empty' is True, there will be
     1521        no ordering in the resulting query (not even the model's default).
     1522        """
     1523        self.order_by = []
     1524        self.extra_order_by = ()
     1525        if force_empty:
     1526            self.default_ordering = False
     1527
     1528    def set_group_by(self):
     1529        """
     1530        Expands the GROUP BY clause required by the query.
     1531
     1532        This will usually be the set of all non-aggregate fields in the
     1533        return data. If the database backend supports grouping by the
     1534        primary key, and the query would be equivalent, the optimization
     1535        will be made automatically.
     1536        """
     1537        self.group_by = []
     1538
     1539        for sel in self.select:
     1540            self.group_by.append(sel)
     1541
     1542    def add_count_column(self):
     1543        """
     1544        Converts the query to do count(...) or count(distinct(pk)) in order to
     1545        get its size.
     1546        """
     1547        if not self.distinct:
     1548            if not self.select:
     1549                count = self.aggregates_module.Count('*', is_summary=True)
     1550            else:
     1551                assert len(self.select) == 1, \
     1552                        "Cannot add count col with multiple cols in 'select': %r" % self.select
     1553                count = self.aggregates_module.Count(self.select[0])
     1554        else:
     1555            opts = self.model._meta
     1556            if not self.select:
     1557                count = self.aggregates_module.Count((self.join((None, opts.db_table, None, None)), opts.pk.column),
     1558                                         is_summary=True, distinct=True)
     1559            else:
     1560                # Because of SQL portability issues, multi-column, distinct
     1561                # counts need a sub-query -- see get_count() for details.
     1562                assert len(self.select) == 1, \
     1563                        "Cannot add count col with multiple cols in 'select'."
     1564
     1565                count = self.aggregates_module.Count(self.select[0], distinct=True)
     1566            # Distinct handling is done in Count(), so don't do it at this
     1567            # level.
     1568            self.distinct = False
     1569
     1570        # Set only aggregate to be the count column.
     1571        # Clear out the select cache to reflect the new unmasked aggregates.
     1572        self.aggregates = {None: count}
     1573        self.set_aggregate_mask(None)
     1574        self.group_by = None
     1575
     1576    def add_select_related(self, fields):
     1577        """
     1578        Sets up the select_related data structure so that we only select
     1579        certain related models (as opposed to all models, when
     1580        self.select_related=True).
     1581        """
     1582        field_dict = {}
     1583        for field in fields:
     1584            d = field_dict
     1585            for part in field.split(LOOKUP_SEP):
     1586                d = d.setdefault(part, {})
     1587        self.select_related = field_dict
     1588        self.related_select_cols = []
     1589        self.related_select_fields = []
     1590
     1591    def add_extra(self, select, select_params, where, params, tables, order_by):
     1592        """
     1593        Adds data to the various extra_* attributes for user-created additions
     1594        to the query.
     1595        """
     1596        if select:
     1597            # We need to pair any placeholder markers in the 'select'
     1598            # dictionary with their parameters in 'select_params' so that
     1599            # subsequent updates to the select dictionary also adjust the
     1600            # parameters appropriately.
     1601            select_pairs = SortedDict()
     1602            if select_params:
     1603                param_iter = iter(select_params)
     1604            else:
     1605                param_iter = iter([])
     1606            for name, entry in select.items():
     1607                entry = force_unicode(entry)
     1608                entry_params = []
     1609                pos = entry.find("%s")
     1610                while pos != -1:
     1611                    entry_params.append(param_iter.next())
     1612                    pos = entry.find("%s", pos + 2)
     1613                select_pairs[name] = (entry, entry_params)
     1614            # This is order preserving, since self.extra_select is a SortedDict.
     1615            self.extra.update(select_pairs)
     1616        if where or params:
     1617            self.where.add(ExtraWhere(where, params), AND)
     1618        if tables:
     1619            self.extra_tables += tuple(tables)
     1620        if order_by:
     1621            self.extra_order_by = order_by
     1622
     1623    def clear_deferred_loading(self):
     1624        """
     1625        Remove any fields from the deferred loading set.
     1626        """
     1627        self.deferred_loading = (set(), True)
     1628
     1629    def add_deferred_loading(self, field_names):
     1630        """
     1631        Add the given list of model field names to the set of fields to
     1632        exclude from loading from the database when automatic column selection
     1633        is done. The new field names are added to any existing field names that
     1634        are deferred (or removed from any existing field names that are marked
     1635        as the only ones for immediate loading).
     1636        """
     1637        # Fields on related models are stored in the literal double-underscore
     1638        # format, so that we can use a set datastructure. We do the foo__bar
     1639        # splitting and handling when computing the SQL colum names (as part of
     1640        # get_columns()).
     1641        existing, defer = self.deferred_loading
     1642        if defer:
     1643            # Add to existing deferred names.
     1644            self.deferred_loading = existing.union(field_names), True
     1645        else:
     1646            # Remove names from the set of any existing "immediate load" names.
     1647            self.deferred_loading = existing.difference(field_names), False
     1648
     1649    def add_immediate_loading(self, field_names):
     1650        """
     1651        Add the given list of model field names to the set of fields to
     1652        retrieve when the SQL is executed ("immediate loading" fields). The
     1653        field names replace any existing immediate loading field names. If
     1654        there are field names already specified for deferred loading, those
     1655        names are removed from the new field_names before storing the new names
     1656        for immediate loading. (That is, immediate loading overrides any
     1657        existing immediate values, but respects existing deferrals.)
     1658        """
     1659        existing, defer = self.deferred_loading
     1660        if defer:
     1661            # Remove any existing deferred names from the current set before
     1662            # setting the new names.
     1663            self.deferred_loading = set(field_names).difference(existing), False
     1664        else:
     1665            # Replace any existing "immediate load" field names.
     1666            self.deferred_loading = set(field_names), False
     1667
     1668    def get_loaded_field_names(self):
     1669        """
     1670        If any fields are marked to be deferred, returns a dictionary mapping
     1671        models to a set of names in those fields that will be loaded. If a
     1672        model is not in the returned dictionary, none of it's fields are
     1673        deferred.
     1674
     1675        If no fields are marked for deferral, returns an empty dictionary.
     1676        """
     1677        collection = {}
     1678        self.deferred_to_data(collection, self.get_loaded_field_names_cb)
     1679        return collection
     1680
     1681    def get_loaded_field_names_cb(self, target, model, fields):
     1682        """
     1683        Callback used by get_deferred_field_names().
     1684        """
     1685        target[model] = set([f.name for f in fields])
     1686
     1687    def set_aggregate_mask(self, names):
     1688        "Set the mask of aggregates that will actually be returned by the SELECT"
     1689        if names is None:
     1690            self.aggregate_select_mask = None
     1691        else:
     1692            self.aggregate_select_mask = set(names)
     1693        self._aggregate_select_cache = None
     1694
     1695    def set_extra_mask(self, names):
     1696        """
     1697        Set the mask of extra select items that will be returned by SELECT,
     1698        we don't actually remove them from the Query since they might be used
     1699        later
     1700        """
     1701        if names is None:
     1702            self.extra_select_mask = None
     1703        else:
     1704            self.extra_select_mask = set(names)
     1705        self._extra_select_cache = None
     1706
     1707    def _aggregate_select(self):
     1708        """The SortedDict of aggregate columns that are not masked, and should
     1709        be used in the SELECT clause.
     1710
     1711        This result is cached for optimization purposes.
     1712        """
     1713        if self._aggregate_select_cache is not None:
     1714            return self._aggregate_select_cache
     1715        elif self.aggregate_select_mask is not None:
     1716            self._aggregate_select_cache = SortedDict([
     1717                (k,v) for k,v in self.aggregates.items()
     1718                if k in self.aggregate_select_mask
     1719            ])
     1720            return self._aggregate_select_cache
     1721        else:
     1722            return self.aggregates
     1723    aggregate_select = property(_aggregate_select)
     1724
     1725    def _extra_select(self):
     1726        if self._extra_select_cache is not None:
     1727            return self._extra_select_cache
     1728        elif self.extra_select_mask is not None:
     1729            self._extra_select_cache = SortedDict([
     1730                (k,v) for k,v in self.extra.items()
     1731                if k in self.extra_select_mask
     1732            ])
     1733            return self._extra_select_cache
     1734        else:
     1735            return self.extra
     1736    extra_select = property(_extra_select)
     1737
     1738    def set_start(self, start):
     1739        """
     1740        Sets the table from which to start joining. The start position is
     1741        specified by the related attribute from the base model. This will
     1742        automatically set to the select column to be the column linked from the
     1743        previous table.
     1744
     1745        This method is primarily for internal use and the error checking isn't
     1746        as friendly as add_filter(). Mostly useful for querying directly
     1747        against the join table of many-to-many relation in a subquery.
     1748        """
     1749        opts = self.model._meta
     1750        alias = self.get_initial_alias()
     1751        field, col, opts, joins, last, extra = self.setup_joins(
     1752                start.split(LOOKUP_SEP), opts, alias, False)
     1753        select_col = self.alias_map[joins[1]][LHS_JOIN_COL]
     1754        select_alias = alias
     1755
     1756        # The call to setup_joins added an extra reference to everything in
     1757        # joins. Reverse that.
     1758        for alias in joins:
     1759            self.unref_alias(alias)
     1760
     1761        # We might be able to trim some joins from the front of this query,
     1762        # providing that we only traverse "always equal" connections (i.e. rhs
     1763        # is *always* the same value as lhs).
     1764        for alias in joins[1:]:
     1765            join_info = self.alias_map[alias]
     1766            if (join_info[LHS_JOIN_COL] != select_col
     1767                    or join_info[JOIN_TYPE] != self.INNER):
     1768                break
     1769            self.unref_alias(select_alias)
     1770            select_alias = join_info[RHS_ALIAS]
     1771            select_col = join_info[RHS_JOIN_COL]
     1772        self.select = [(select_alias, select_col)]
     1773        self.remove_inherited_models()
     1774
     1775
     1776def get_order_dir(field, default='ASC'):
     1777    """
     1778    Returns the field name and direction for an order specification. For
     1779    example, '-foo' is returned as ('foo', 'DESC').
     1780
     1781    The 'default' param is used to indicate which way no prefix (or a '+'
     1782    prefix) should sort. The '-' prefix always sorts the opposite way.
     1783    """
     1784    dirn = ORDER_DIR[default]
     1785    if field[0] == '-':
     1786        return field[1:], dirn[1]
     1787    return field, dirn[0]
     1788
     1789
     1790def setup_join_cache(sender, **kwargs):
     1791    """
     1792    The information needed to join between model fields is something that is
     1793    invariant over the life of the model, so we cache it in the model's Options
     1794    class, rather than recomputing it all the time.
     1795
     1796    This method initialises the (empty) cache when the model is created.
     1797    """
     1798    sender._meta._join_cache = {}
     1799
     1800signals.class_prepared.connect(setup_join_cache)
     1801
     1802def add_to_dict(data, key, value):
     1803    """
     1804    A helper function to add "value" to the set of values for "key", whether or
     1805    not "key" already exists.
     1806    """
     1807    if key in data:
     1808        data[key].add(value)
     1809    else:
     1810        data[key] = set([value])
     1811
     1812def get_proxied_model(opts):
     1813    int_opts = opts
     1814    proxied_model = None
     1815    while int_opts.proxy:
     1816        proxied_model = int_opts.proxy_for_model
     1817        int_opts = proxied_model._meta
     1818    return proxied_model
Back to Top