| | 1 | """ |
| | 2 | Create SQL statements for QuerySets. |
| | 3 | |
| | 4 | The code in here encapsulates all of the SQL construction so that QuerySets |
| | 5 | themselves do not have to (and could be backed by things other than SQL |
| | 6 | databases). The abstraction barrier only works one way: this module has to know |
| | 7 | all about the internals of models in order to get the information it needs. |
| | 8 | """ |
| | 9 | |
| | 10 | from django.utils.copycompat import deepcopy |
| | 11 | from django.utils.tree import Node |
| | 12 | from django.utils.datastructures import SortedDict |
| | 13 | from django.utils.encoding import force_unicode |
| | 14 | from django.db import connections, DEFAULT_DB_ALIAS |
| | 15 | from django.db.models import signals |
| | 16 | from django.db.models.fields import FieldDoesNotExist |
| | 17 | from django.db.models.query_utils import select_related_descend, InvalidQuery |
| | 18 | from django.db.models.sql import aggregates as base_aggregates_module |
| | 19 | from django.db.models.sql.constants import * |
| | 20 | from django.db.models.sql.datastructures import EmptyResultSet, Empty, MultiJoin |
| | 21 | from django.db.models.sql.expressions import SQLEvaluator |
| | 22 | from django.db.models.sql.where import (WhereNode, Constraint, EverythingNode, |
| | 23 | ExtraWhere, AND, OR) |
| | 24 | from django.core.exceptions import FieldError |
| | 25 | |
| | 26 | __all__ = ['Query', 'RawQuery'] |
| | 27 | |
| | 28 | class RawQuery(object): |
| | 29 | """ |
| | 30 | A single raw SQL query |
| | 31 | """ |
| | 32 | |
| | 33 | def __init__(self, sql, using, params=None): |
| | 34 | self.validate_sql(sql) |
| | 35 | self.params = params or () |
| | 36 | self.sql = sql |
| | 37 | self.using = using |
| | 38 | self.cursor = None |
| | 39 | |
| | 40 | def clone(self, using): |
| | 41 | return RawQuery(self.sql, using, params=self.params) |
| | 42 | |
| | 43 | def get_columns(self): |
| | 44 | if self.cursor is None: |
| | 45 | self._execute_query() |
| | 46 | converter = connections[self.using].introspection.table_name_converter |
| | 47 | return [converter(column_meta[0]) |
| | 48 | for column_meta in self.cursor.description] |
| | 49 | |
| | 50 | def validate_sql(self, sql): |
| | 51 | if not sql.lower().strip().startswith('select'): |
| | 52 | raise InvalidQuery('Raw queries are limited to SELECT queries. Use ' |
| | 53 | 'connection.cursor directly for other types of queries.') |
| | 54 | |
| | 55 | def __iter__(self): |
| | 56 | # Always execute a new query for a new iterator. |
| | 57 | # This could be optimized with a cache at the expense of RAM. |
| | 58 | self._execute_query() |
| | 59 | return iter(self.cursor) |
| | 60 | |
| | 61 | def __repr__(self): |
| | 62 | return "<RawQuery: %r>" % (self.sql % self.params) |
| | 63 | |
| | 64 | def _execute_query(self): |
| | 65 | self.cursor = connections[self.using].cursor() |
| | 66 | self.cursor.execute(self.sql, self.params) |
| | 67 | |
| | 68 | |
| | 69 | class Query(object): |
| | 70 | """ |
| | 71 | A single SQL query. |
| | 72 | """ |
| | 73 | # SQL join types. These are part of the class because their string forms |
| | 74 | # vary from database to database and can be customised by a subclass. |
| | 75 | INNER = 'INNER JOIN' |
| | 76 | LOUTER = 'LEFT OUTER JOIN' |
| | 77 | |
| | 78 | alias_prefix = 'T' |
| | 79 | query_terms = QUERY_TERMS |
| | 80 | aggregates_module = base_aggregates_module |
| | 81 | |
| | 82 | compiler = 'SQLCompiler' |
| | 83 | |
| | 84 | def __init__(self, model, where=WhereNode): |
| | 85 | self.model = model |
| | 86 | self.alias_refcount = {} |
| | 87 | self.alias_map = {} # Maps alias to join information |
| | 88 | self.table_map = {} # Maps table names to list of aliases. |
| | 89 | self.join_map = {} |
| | 90 | self.rev_join_map = {} # Reverse of join_map. |
| | 91 | self.quote_cache = {} |
| | 92 | self.default_cols = True |
| | 93 | self.default_ordering = True |
| | 94 | self.standard_ordering = True |
| | 95 | self.ordering_aliases = [] |
| | 96 | self.select_fields = [] |
| | 97 | self.related_select_fields = [] |
| | 98 | self.dupe_avoidance = {} |
| | 99 | self.used_aliases = set() |
| | 100 | self.filter_is_sticky = False |
| | 101 | self.included_inherited_models = {} |
| | 102 | |
| | 103 | # SQL-related attributes |
| | 104 | self.select = [] |
| | 105 | self.tables = [] # Aliases in the order they are created. |
| | 106 | self.where = where() |
| | 107 | self.where_class = where |
| | 108 | self.group_by = None |
| | 109 | self.having = where() |
| | 110 | self.order_by = [] |
| | 111 | self.low_mark, self.high_mark = 0, None # Used for offset/limit |
| | 112 | self.distinct = False |
| | 113 | self.select_related = False |
| | 114 | self.related_select_cols = [] |
| | 115 | |
| | 116 | # SQL aggregate-related attributes |
| | 117 | self.aggregates = SortedDict() # Maps alias -> SQL aggregate function |
| | 118 | self.aggregate_select_mask = None |
| | 119 | self._aggregate_select_cache = None |
| | 120 | |
| | 121 | # Arbitrary maximum limit for select_related. Prevents infinite |
| | 122 | # recursion. Can be changed by the depth parameter to select_related(). |
| | 123 | self.max_depth = 5 |
| | 124 | |
| | 125 | # These are for extensions. The contents are more or less appended |
| | 126 | # verbatim to the appropriate clause. |
| | 127 | self.extra = SortedDict() # Maps col_alias -> (col_sql, params). |
| | 128 | self.extra_select_mask = None |
| | 129 | self._extra_select_cache = None |
| | 130 | |
| | 131 | self.extra_tables = () |
| | 132 | self.extra_order_by = () |
| | 133 | |
| | 134 | # A tuple that is a set of model field names and either True, if these |
| | 135 | # are the fields to defer, or False if these are the only fields to |
| | 136 | # load. |
| | 137 | self.deferred_loading = (set(), True) |
| | 138 | |
| | 139 | def __str__(self): |
| | 140 | """ |
| | 141 | Returns the query as a string of SQL with the parameter values |
| | 142 | substituted in. |
| | 143 | |
| | 144 | Parameter values won't necessarily be quoted correctly, since that is |
| | 145 | done by the database interface at execution time. |
| | 146 | """ |
| | 147 | sql, params = self.get_compiler(DEFAULT_DB_ALIAS).as_sql() |
| | 148 | return sql % params |
| | 149 | |
| | 150 | def __deepcopy__(self, memo): |
| | 151 | result = self.clone(memo=memo) |
| | 152 | memo[id(self)] = result |
| | 153 | return result |
| | 154 | |
| | 155 | def __getstate__(self): |
| | 156 | """ |
| | 157 | Pickling support. |
| | 158 | """ |
| | 159 | obj_dict = self.__dict__.copy() |
| | 160 | obj_dict['related_select_fields'] = [] |
| | 161 | obj_dict['related_select_cols'] = [] |
| | 162 | |
| | 163 | # Fields can't be pickled, so if a field list has been |
| | 164 | # specified, we pickle the list of field names instead. |
| | 165 | # None is also a possible value; that can pass as-is |
| | 166 | obj_dict['select_fields'] = [ |
| | 167 | f is not None and f.name or None |
| | 168 | for f in obj_dict['select_fields'] |
| | 169 | ] |
| | 170 | return obj_dict |
| | 171 | |
| | 172 | def __setstate__(self, obj_dict): |
| | 173 | """ |
| | 174 | Unpickling support. |
| | 175 | """ |
| | 176 | # Rebuild list of field instances |
| | 177 | obj_dict['select_fields'] = [ |
| | 178 | name is not None and obj_dict['model']._meta.get_field(name) or None |
| | 179 | for name in obj_dict['select_fields'] |
| | 180 | ] |
| | 181 | |
| | 182 | self.__dict__.update(obj_dict) |
| | 183 | |
| | 184 | def prepare(self): |
| | 185 | return self |
| | 186 | |
| | 187 | def get_compiler(self, using=None, connection=None): |
| | 188 | if using is None and connection is None: |
| | 189 | raise ValueError("Need either using or connection") |
| | 190 | if using: |
| | 191 | connection = connections[using] |
| | 192 | |
| | 193 | # Check that the compiler will be able to execute the query |
| | 194 | for alias, aggregate in self.aggregate_select.items(): |
| | 195 | connection.ops.check_aggregate_support(aggregate) |
| | 196 | |
| | 197 | return connection.ops.compiler(self.compiler)(self, connection, using) |
| | 198 | |
| | 199 | def get_meta(self): |
| | 200 | """ |
| | 201 | Returns the Options instance (the model._meta) from which to start |
| | 202 | processing. Normally, this is self.model._meta, but it can be changed |
| | 203 | by subclasses. |
| | 204 | """ |
| | 205 | return self.model._meta |
| | 206 | |
| | 207 | def clone(self, klass=None, memo=None, **kwargs): |
| | 208 | """ |
| | 209 | Creates a copy of the current instance. The 'kwargs' parameter can be |
| | 210 | used by clients to update attributes after copying has taken place. |
| | 211 | """ |
| | 212 | obj = Empty() |
| | 213 | obj.__class__ = klass or self.__class__ |
| | 214 | obj.model = self.model |
| | 215 | obj.alias_refcount = self.alias_refcount.copy() |
| | 216 | obj.alias_map = self.alias_map.copy() |
| | 217 | obj.table_map = self.table_map.copy() |
| | 218 | obj.join_map = self.join_map.copy() |
| | 219 | obj.rev_join_map = self.rev_join_map.copy() |
| | 220 | obj.quote_cache = {} |
| | 221 | obj.default_cols = self.default_cols |
| | 222 | obj.default_ordering = self.default_ordering |
| | 223 | obj.standard_ordering = self.standard_ordering |
| | 224 | obj.included_inherited_models = self.included_inherited_models.copy() |
| | 225 | obj.ordering_aliases = [] |
| | 226 | obj.select_fields = self.select_fields[:] |
| | 227 | obj.related_select_fields = self.related_select_fields[:] |
| | 228 | obj.dupe_avoidance = self.dupe_avoidance.copy() |
| | 229 | obj.select = self.select[:] |
| | 230 | obj.tables = self.tables[:] |
| | 231 | obj.where = deepcopy(self.where, memo=memo) |
| | 232 | obj.where_class = self.where_class |
| | 233 | if self.group_by is None: |
| | 234 | obj.group_by = None |
| | 235 | else: |
| | 236 | obj.group_by = self.group_by[:] |
| | 237 | obj.having = deepcopy(self.having, memo=memo) |
| | 238 | obj.order_by = self.order_by[:] |
| | 239 | obj.low_mark, obj.high_mark = self.low_mark, self.high_mark |
| | 240 | obj.distinct = self.distinct |
| | 241 | obj.select_related = self.select_related |
| | 242 | obj.related_select_cols = [] |
| | 243 | obj.aggregates = deepcopy(self.aggregates, memo=memo) |
| | 244 | if self.aggregate_select_mask is None: |
| | 245 | obj.aggregate_select_mask = None |
| | 246 | else: |
| | 247 | obj.aggregate_select_mask = self.aggregate_select_mask.copy() |
| | 248 | # _aggregate_select_cache cannot be copied, as doing so breaks the |
| | 249 | # (necessary) state in which both aggregates and |
| | 250 | # _aggregate_select_cache point to the same underlying objects. |
| | 251 | # It will get re-populated in the cloned queryset the next time it's |
| | 252 | # used. |
| | 253 | obj._aggregate_select_cache = None |
| | 254 | obj.max_depth = self.max_depth |
| | 255 | obj.extra = self.extra.copy() |
| | 256 | if self.extra_select_mask is None: |
| | 257 | obj.extra_select_mask = None |
| | 258 | else: |
| | 259 | obj.extra_select_mask = self.extra_select_mask.copy() |
| | 260 | if self._extra_select_cache is None: |
| | 261 | obj._extra_select_cache = None |
| | 262 | else: |
| | 263 | obj._extra_select_cache = self._extra_select_cache.copy() |
| | 264 | obj.extra_tables = self.extra_tables |
| | 265 | obj.extra_order_by = self.extra_order_by |
| | 266 | obj.deferred_loading = deepcopy(self.deferred_loading, memo=memo) |
| | 267 | if self.filter_is_sticky and self.used_aliases: |
| | 268 | obj.used_aliases = self.used_aliases.copy() |
| | 269 | else: |
| | 270 | obj.used_aliases = set() |
| | 271 | obj.filter_is_sticky = False |
| | 272 | obj.__dict__.update(kwargs) |
| | 273 | if hasattr(obj, '_setup_query'): |
| | 274 | obj._setup_query() |
| | 275 | return obj |
| | 276 | |
| | 277 | def convert_values(self, value, field, connection): |
| | 278 | """Convert the database-returned value into a type that is consistent |
| | 279 | across database backends. |
| | 280 | |
| | 281 | By default, this defers to the underlying backend operations, but |
| | 282 | it can be overridden by Query classes for specific backends. |
| | 283 | """ |
| | 284 | return connection.ops.convert_values(value, field) |
| | 285 | |
| | 286 | def resolve_aggregate(self, value, aggregate, connection): |
| | 287 | """Resolve the value of aggregates returned by the database to |
| | 288 | consistent (and reasonable) types. |
| | 289 | |
| | 290 | This is required because of the predisposition of certain backends |
| | 291 | to return Decimal and long types when they are not needed. |
| | 292 | """ |
| | 293 | if value is None: |
| | 294 | if aggregate.is_ordinal: |
| | 295 | return 0 |
| | 296 | # Return None as-is |
| | 297 | return value |
| | 298 | elif aggregate.is_ordinal: |
| | 299 | # Any ordinal aggregate (e.g., count) returns an int |
| | 300 | return int(value) |
| | 301 | elif aggregate.is_computed: |
| | 302 | # Any computed aggregate (e.g., avg) returns a float |
| | 303 | return float(value) |
| | 304 | else: |
| | 305 | # Return value depends on the type of the field being processed. |
| | 306 | return self.convert_values(value, aggregate.field, connection) |
| | 307 | |
| | 308 | def get_aggregation(self, using): |
| | 309 | """ |
| | 310 | Returns the dictionary with the values of the existing aggregations. |
| | 311 | """ |
| | 312 | if not self.aggregate_select: |
| | 313 | return {} |
| | 314 | |
| | 315 | # If there is a group by clause, aggregating does not add useful |
| | 316 | # information but retrieves only the first row. Aggregate |
| | 317 | # over the subquery instead. |
| | 318 | if self.group_by is not None: |
| | 319 | from subqueries import AggregateQuery |
| | 320 | query = AggregateQuery(self.model) |
| | 321 | |
| | 322 | obj = self.clone() |
| | 323 | |
| | 324 | # Remove any aggregates marked for reduction from the subquery |
| | 325 | # and move them to the outer AggregateQuery. |
| | 326 | for alias, aggregate in self.aggregate_select.items(): |
| | 327 | if aggregate.is_summary: |
| | 328 | query.aggregate_select[alias] = aggregate |
| | 329 | del obj.aggregate_select[alias] |
| | 330 | |
| | 331 | query.add_subquery(obj, using) |
| | 332 | else: |
| | 333 | query = self |
| | 334 | self.select = [] |
| | 335 | self.default_cols = False |
| | 336 | self.extra = {} |
| | 337 | self.remove_inherited_models() |
| | 338 | |
| | 339 | query.clear_ordering(True) |
| | 340 | query.clear_limits() |
| | 341 | query.select_related = False |
| | 342 | query.related_select_cols = [] |
| | 343 | query.related_select_fields = [] |
| | 344 | |
| | 345 | result = query.get_compiler(using).execute_sql(SINGLE) |
| | 346 | if result is None: |
| | 347 | result = [None for q in query.aggregate_select.items()] |
| | 348 | |
| | 349 | return dict([ |
| | 350 | (alias, self.resolve_aggregate(val, aggregate, connection=connections[using])) |
| | 351 | for (alias, aggregate), val |
| | 352 | in zip(query.aggregate_select.items(), result) |
| | 353 | ]) |
| | 354 | |
| | 355 | def get_count(self, using): |
| | 356 | """ |
| | 357 | Performs a COUNT() query using the current filter constraints. |
| | 358 | """ |
| | 359 | obj = self.clone() |
| | 360 | if len(self.select) > 1 or self.aggregate_select: |
| | 361 | # If a select clause exists, then the query has already started to |
| | 362 | # specify the columns that are to be returned. |
| | 363 | # In this case, we need to use a subquery to evaluate the count. |
| | 364 | from subqueries import AggregateQuery |
| | 365 | subquery = obj |
| | 366 | subquery.clear_ordering(True) |
| | 367 | subquery.clear_limits() |
| | 368 | |
| | 369 | obj = AggregateQuery(obj.model) |
| | 370 | obj.add_subquery(subquery, using=using) |
| | 371 | |
| | 372 | obj.add_count_column() |
| | 373 | number = obj.get_aggregation(using=using)[None] |
| | 374 | |
| | 375 | # Apply offset and limit constraints manually, since using LIMIT/OFFSET |
| | 376 | # in SQL (in variants that provide them) doesn't change the COUNT |
| | 377 | # output. |
| | 378 | number = max(0, number - self.low_mark) |
| | 379 | if self.high_mark is not None: |
| | 380 | number = min(number, self.high_mark - self.low_mark) |
| | 381 | |
| | 382 | return number |
| | 383 | |
| | 384 | def has_results(self, using): |
| | 385 | q = self.clone() |
| | 386 | q.add_extra({'a': 1}, None, None, None, None, None) |
| | 387 | q.select = [] |
| | 388 | q.select_fields = [] |
| | 389 | q.default_cols = False |
| | 390 | q.select_related = False |
| | 391 | q.set_extra_mask(('a',)) |
| | 392 | q.set_aggregate_mask(()) |
| | 393 | q.clear_ordering(True) |
| | 394 | q.set_limits(high=1) |
| | 395 | compiler = q.get_compiler(using=using) |
| | 396 | return bool(compiler.execute_sql(SINGLE)) |
| | 397 | |
| | 398 | def combine(self, rhs, connector): |
| | 399 | """ |
| | 400 | Merge the 'rhs' query into the current one (with any 'rhs' effects |
| | 401 | being applied *after* (that is, "to the right of") anything in the |
| | 402 | current query. 'rhs' is not modified during a call to this function. |
| | 403 | |
| | 404 | The 'connector' parameter describes how to connect filters from the |
| | 405 | 'rhs' query. |
| | 406 | """ |
| | 407 | assert self.model == rhs.model, \ |
| | 408 | "Cannot combine queries on two different base models." |
| | 409 | assert self.can_filter(), \ |
| | 410 | "Cannot combine queries once a slice has been taken." |
| | 411 | assert self.distinct == rhs.distinct, \ |
| | 412 | "Cannot combine a unique query with a non-unique query." |
| | 413 | |
| | 414 | self.remove_inherited_models() |
| | 415 | # Work out how to relabel the rhs aliases, if necessary. |
| | 416 | change_map = {} |
| | 417 | used = set() |
| | 418 | conjunction = (connector == AND) |
| | 419 | first = True |
| | 420 | for alias in rhs.tables: |
| | 421 | if not rhs.alias_refcount[alias]: |
| | 422 | # An unused alias. |
| | 423 | continue |
| | 424 | promote = (rhs.alias_map[alias][JOIN_TYPE] == self.LOUTER) |
| | 425 | new_alias = self.join(rhs.rev_join_map[alias], |
| | 426 | (conjunction and not first), used, promote, not conjunction) |
| | 427 | used.add(new_alias) |
| | 428 | change_map[alias] = new_alias |
| | 429 | first = False |
| | 430 | |
| | 431 | # So that we don't exclude valid results in an "or" query combination, |
| | 432 | # the first join that is exclusive to the lhs (self) must be converted |
| | 433 | # to an outer join. |
| | 434 | if not conjunction: |
| | 435 | for alias in self.tables[1:]: |
| | 436 | if self.alias_refcount[alias] == 1: |
| | 437 | self.promote_alias(alias, True) |
| | 438 | break |
| | 439 | |
| | 440 | # Now relabel a copy of the rhs where-clause and add it to the current |
| | 441 | # one. |
| | 442 | if rhs.where: |
| | 443 | w = deepcopy(rhs.where) |
| | 444 | w.relabel_aliases(change_map) |
| | 445 | if not self.where: |
| | 446 | # Since 'self' matches everything, add an explicit "include |
| | 447 | # everything" where-constraint so that connections between the |
| | 448 | # where clauses won't exclude valid results. |
| | 449 | self.where.add(EverythingNode(), AND) |
| | 450 | elif self.where: |
| | 451 | # rhs has an empty where clause. |
| | 452 | w = self.where_class() |
| | 453 | w.add(EverythingNode(), AND) |
| | 454 | else: |
| | 455 | w = self.where_class() |
| | 456 | self.where.add(w, connector) |
| | 457 | |
| | 458 | # Selection columns and extra extensions are those provided by 'rhs'. |
| | 459 | self.select = [] |
| | 460 | for col in rhs.select: |
| | 461 | if isinstance(col, (list, tuple)): |
| | 462 | self.select.append((change_map.get(col[0], col[0]), col[1])) |
| | 463 | else: |
| | 464 | item = deepcopy(col) |
| | 465 | item.relabel_aliases(change_map) |
| | 466 | self.select.append(item) |
| | 467 | self.select_fields = rhs.select_fields[:] |
| | 468 | |
| | 469 | if connector == OR: |
| | 470 | # It would be nice to be able to handle this, but the queries don't |
| | 471 | # really make sense (or return consistent value sets). Not worth |
| | 472 | # the extra complexity when you can write a real query instead. |
| | 473 | if self.extra and rhs.extra: |
| | 474 | raise ValueError("When merging querysets using 'or', you " |
| | 475 | "cannot have extra(select=...) on both sides.") |
| | 476 | self.extra.update(rhs.extra) |
| | 477 | extra_select_mask = set() |
| | 478 | if self.extra_select_mask is not None: |
| | 479 | extra_select_mask.update(self.extra_select_mask) |
| | 480 | if rhs.extra_select_mask is not None: |
| | 481 | extra_select_mask.update(rhs.extra_select_mask) |
| | 482 | if extra_select_mask: |
| | 483 | self.set_extra_mask(extra_select_mask) |
| | 484 | self.extra_tables += rhs.extra_tables |
| | 485 | |
| | 486 | # Ordering uses the 'rhs' ordering, unless it has none, in which case |
| | 487 | # the current ordering is used. |
| | 488 | self.order_by = rhs.order_by and rhs.order_by[:] or self.order_by |
| | 489 | self.extra_order_by = rhs.extra_order_by or self.extra_order_by |
| | 490 | |
| | 491 | def deferred_to_data(self, target, callback): |
| | 492 | """ |
| | 493 | Converts the self.deferred_loading data structure to an alternate data |
| | 494 | structure, describing the field that *will* be loaded. This is used to |
| | 495 | compute the columns to select from the database and also by the |
| | 496 | QuerySet class to work out which fields are being initialised on each |
| | 497 | model. Models that have all their fields included aren't mentioned in |
| | 498 | the result, only those that have field restrictions in place. |
| | 499 | |
| | 500 | The "target" parameter is the instance that is populated (in place). |
| | 501 | The "callback" is a function that is called whenever a (model, field) |
| | 502 | pair need to be added to "target". It accepts three parameters: |
| | 503 | "target", and the model and list of fields being added for that model. |
| | 504 | """ |
| | 505 | field_names, defer = self.deferred_loading |
| | 506 | if not field_names: |
| | 507 | return |
| | 508 | columns = set() |
| | 509 | orig_opts = self.model._meta |
| | 510 | seen = {} |
| | 511 | must_include = {self.model: set([orig_opts.pk])} |
| | 512 | for field_name in field_names: |
| | 513 | parts = field_name.split(LOOKUP_SEP) |
| | 514 | cur_model = self.model |
| | 515 | opts = orig_opts |
| | 516 | for name in parts[:-1]: |
| | 517 | old_model = cur_model |
| | 518 | source = opts.get_field_by_name(name)[0] |
| | 519 | cur_model = opts.get_field_by_name(name)[0].rel.to |
| | 520 | opts = cur_model._meta |
| | 521 | # Even if we're "just passing through" this model, we must add |
| | 522 | # both the current model's pk and the related reference field |
| | 523 | # to the things we select. |
| | 524 | must_include[old_model].add(source) |
| | 525 | add_to_dict(must_include, cur_model, opts.pk) |
| | 526 | field, model, _, _ = opts.get_field_by_name(parts[-1]) |
| | 527 | if model is None: |
| | 528 | model = cur_model |
| | 529 | add_to_dict(seen, model, field) |
| | 530 | |
| | 531 | if defer: |
| | 532 | # We need to load all fields for each model, except those that |
| | 533 | # appear in "seen" (for all models that appear in "seen"). The only |
| | 534 | # slight complexity here is handling fields that exist on parent |
| | 535 | # models. |
| | 536 | workset = {} |
| | 537 | for model, values in seen.iteritems(): |
| | 538 | for field in model._meta.local_fields: |
| | 539 | if field in values: |
| | 540 | continue |
| | 541 | add_to_dict(workset, model, field) |
| | 542 | for model, values in must_include.iteritems(): |
| | 543 | # If we haven't included a model in workset, we don't add the |
| | 544 | # corresponding must_include fields for that model, since an |
| | 545 | # empty set means "include all fields". That's why there's no |
| | 546 | # "else" branch here. |
| | 547 | if model in workset: |
| | 548 | workset[model].update(values) |
| | 549 | for model, values in workset.iteritems(): |
| | 550 | callback(target, model, values) |
| | 551 | else: |
| | 552 | for model, values in must_include.iteritems(): |
| | 553 | if model in seen: |
| | 554 | seen[model].update(values) |
| | 555 | else: |
| | 556 | # As we've passed through this model, but not explicitly |
| | 557 | # included any fields, we have to make sure it's mentioned |
| | 558 | # so that only the "must include" fields are pulled in. |
| | 559 | seen[model] = values |
| | 560 | # Now ensure that every model in the inheritance chain is mentioned |
| | 561 | # in the parent list. Again, it must be mentioned to ensure that |
| | 562 | # only "must include" fields are pulled in. |
| | 563 | for model in orig_opts.get_parent_list(): |
| | 564 | if model not in seen: |
| | 565 | seen[model] = set() |
| | 566 | for model, values in seen.iteritems(): |
| | 567 | callback(target, model, values) |
| | 568 | |
| | 569 | |
| | 570 | def deferred_to_columns_cb(self, target, model, fields): |
| | 571 | """ |
| | 572 | Callback used by deferred_to_columns(). The "target" parameter should |
| | 573 | be a set instance. |
| | 574 | """ |
| | 575 | table = model._meta.db_table |
| | 576 | if table not in target: |
| | 577 | target[table] = set() |
| | 578 | for field in fields: |
| | 579 | target[table].add(field.column) |
| | 580 | |
| | 581 | |
| | 582 | def table_alias(self, table_name, create=False): |
| | 583 | """ |
| | 584 | Returns a table alias for the given table_name and whether this is a |
| | 585 | new alias or not. |
| | 586 | |
| | 587 | If 'create' is true, a new alias is always created. Otherwise, the |
| | 588 | most recently created alias for the table (if one exists) is reused. |
| | 589 | """ |
| | 590 | current = self.table_map.get(table_name) |
| | 591 | if not create and current: |
| | 592 | alias = current[0] |
| | 593 | self.alias_refcount[alias] += 1 |
| | 594 | return alias, False |
| | 595 | |
| | 596 | # Create a new alias for this table. |
| | 597 | if current: |
| | 598 | alias = '%s%d' % (self.alias_prefix, len(self.alias_map) + 1) |
| | 599 | current.append(alias) |
| | 600 | else: |
| | 601 | # The first occurence of a table uses the table name directly. |
| | 602 | alias = table_name |
| | 603 | self.table_map[alias] = [alias] |
| | 604 | self.alias_refcount[alias] = 1 |
| | 605 | self.tables.append(alias) |
| | 606 | return alias, True |
| | 607 | |
| | 608 | def ref_alias(self, alias): |
| | 609 | """ Increases the reference count for this alias. """ |
| | 610 | self.alias_refcount[alias] += 1 |
| | 611 | |
| | 612 | def unref_alias(self, alias): |
| | 613 | """ Decreases the reference count for this alias. """ |
| | 614 | self.alias_refcount[alias] -= 1 |
| | 615 | |
| | 616 | def promote_alias(self, alias, unconditional=False): |
| | 617 | """ |
| | 618 | Promotes the join type of an alias to an outer join if it's possible |
| | 619 | for the join to contain NULL values on the left. If 'unconditional' is |
| | 620 | False, the join is only promoted if it is nullable, otherwise it is |
| | 621 | always promoted. |
| | 622 | |
| | 623 | Returns True if the join was promoted. |
| | 624 | """ |
| | 625 | if ((unconditional or self.alias_map[alias][NULLABLE]) and |
| | 626 | self.alias_map[alias][JOIN_TYPE] != self.LOUTER): |
| | 627 | data = list(self.alias_map[alias]) |
| | 628 | data[JOIN_TYPE] = self.LOUTER |
| | 629 | self.alias_map[alias] = tuple(data) |
| | 630 | return True |
| | 631 | return False |
| | 632 | |
| | 633 | def promote_alias_chain(self, chain, must_promote=False): |
| | 634 | """ |
| | 635 | Walks along a chain of aliases, promoting the first nullable join and |
| | 636 | any joins following that. If 'must_promote' is True, all the aliases in |
| | 637 | the chain are promoted. |
| | 638 | """ |
| | 639 | for alias in chain: |
| | 640 | if self.promote_alias(alias, must_promote): |
| | 641 | must_promote = True |
| | 642 | |
| | 643 | def promote_unused_aliases(self, initial_refcounts, used_aliases): |
| | 644 | """ |
| | 645 | Given a "before" copy of the alias_refcounts dictionary (as |
| | 646 | 'initial_refcounts') and a collection of aliases that may have been |
| | 647 | changed or created, works out which aliases have been created since |
| | 648 | then and which ones haven't been used and promotes all of those |
| | 649 | aliases, plus any children of theirs in the alias tree, to outer joins. |
| | 650 | """ |
| | 651 | # FIXME: There's some (a lot of!) overlap with the similar OR promotion |
| | 652 | # in add_filter(). It's not quite identical, but is very similar. So |
| | 653 | # pulling out the common bits is something for later. |
| | 654 | considered = {} |
| | 655 | for alias in self.tables: |
| | 656 | if alias not in used_aliases: |
| | 657 | continue |
| | 658 | if (alias not in initial_refcounts or |
| | 659 | self.alias_refcount[alias] == initial_refcounts[alias]): |
| | 660 | parent = self.alias_map[alias][LHS_ALIAS] |
| | 661 | must_promote = considered.get(parent, False) |
| | 662 | promoted = self.promote_alias(alias, must_promote) |
| | 663 | considered[alias] = must_promote or promoted |
| | 664 | |
| | 665 | def change_aliases(self, change_map): |
| | 666 | """ |
| | 667 | Changes the aliases in change_map (which maps old-alias -> new-alias), |
| | 668 | relabelling any references to them in select columns and the where |
| | 669 | clause. |
| | 670 | """ |
| | 671 | assert set(change_map.keys()).intersection(set(change_map.values())) == set() |
| | 672 | |
| | 673 | # 1. Update references in "select" (normal columns plus aliases), |
| | 674 | # "group by", "where" and "having". |
| | 675 | self.where.relabel_aliases(change_map) |
| | 676 | self.having.relabel_aliases(change_map) |
| | 677 | for columns in (self.select, self.aggregates.values(), self.group_by or []): |
| | 678 | for pos, col in enumerate(columns): |
| | 679 | if isinstance(col, (list, tuple)): |
| | 680 | old_alias = col[0] |
| | 681 | columns[pos] = (change_map.get(old_alias, old_alias), col[1]) |
| | 682 | else: |
| | 683 | col.relabel_aliases(change_map) |
| | 684 | |
| | 685 | # 2. Rename the alias in the internal table/alias datastructures. |
| | 686 | for old_alias, new_alias in change_map.iteritems(): |
| | 687 | alias_data = list(self.alias_map[old_alias]) |
| | 688 | alias_data[RHS_ALIAS] = new_alias |
| | 689 | |
| | 690 | t = self.rev_join_map[old_alias] |
| | 691 | data = list(self.join_map[t]) |
| | 692 | data[data.index(old_alias)] = new_alias |
| | 693 | self.join_map[t] = tuple(data) |
| | 694 | self.rev_join_map[new_alias] = t |
| | 695 | del self.rev_join_map[old_alias] |
| | 696 | self.alias_refcount[new_alias] = self.alias_refcount[old_alias] |
| | 697 | del self.alias_refcount[old_alias] |
| | 698 | self.alias_map[new_alias] = tuple(alias_data) |
| | 699 | del self.alias_map[old_alias] |
| | 700 | |
| | 701 | table_aliases = self.table_map[alias_data[TABLE_NAME]] |
| | 702 | for pos, alias in enumerate(table_aliases): |
| | 703 | if alias == old_alias: |
| | 704 | table_aliases[pos] = new_alias |
| | 705 | break |
| | 706 | for pos, alias in enumerate(self.tables): |
| | 707 | if alias == old_alias: |
| | 708 | self.tables[pos] = new_alias |
| | 709 | break |
| | 710 | for key, alias in self.included_inherited_models.items(): |
| | 711 | if alias in change_map: |
| | 712 | self.included_inherited_models[key] = change_map[alias] |
| | 713 | |
| | 714 | # 3. Update any joins that refer to the old alias. |
| | 715 | for alias, data in self.alias_map.iteritems(): |
| | 716 | lhs = data[LHS_ALIAS] |
| | 717 | if lhs in change_map: |
| | 718 | data = list(data) |
| | 719 | data[LHS_ALIAS] = change_map[lhs] |
| | 720 | self.alias_map[alias] = tuple(data) |
| | 721 | |
| | 722 | def bump_prefix(self, exceptions=()): |
| | 723 | """ |
| | 724 | Changes the alias prefix to the next letter in the alphabet and |
| | 725 | relabels all the aliases. Even tables that previously had no alias will |
| | 726 | get an alias after this call (it's mostly used for nested queries and |
| | 727 | the outer query will already be using the non-aliased table name). |
| | 728 | |
| | 729 | Subclasses who create their own prefix should override this method to |
| | 730 | produce a similar result (a new prefix and relabelled aliases). |
| | 731 | |
| | 732 | The 'exceptions' parameter is a container that holds alias names which |
| | 733 | should not be changed. |
| | 734 | """ |
| | 735 | current = ord(self.alias_prefix) |
| | 736 | assert current < ord('Z') |
| | 737 | prefix = chr(current + 1) |
| | 738 | self.alias_prefix = prefix |
| | 739 | change_map = {} |
| | 740 | for pos, alias in enumerate(self.tables): |
| | 741 | if alias in exceptions: |
| | 742 | continue |
| | 743 | new_alias = '%s%d' % (prefix, pos) |
| | 744 | change_map[alias] = new_alias |
| | 745 | self.tables[pos] = new_alias |
| | 746 | self.change_aliases(change_map) |
| | 747 | |
| | 748 | def get_initial_alias(self): |
| | 749 | """ |
| | 750 | Returns the first alias for this query, after increasing its reference |
| | 751 | count. |
| | 752 | """ |
| | 753 | if self.tables: |
| | 754 | alias = self.tables[0] |
| | 755 | self.ref_alias(alias) |
| | 756 | else: |
| | 757 | alias = self.join((None, self.model._meta.db_table, None, None)) |
| | 758 | return alias |
| | 759 | |
| | 760 | def count_active_tables(self): |
| | 761 | """ |
| | 762 | Returns the number of tables in this query with a non-zero reference |
| | 763 | count. |
| | 764 | """ |
| | 765 | return len([1 for count in self.alias_refcount.itervalues() if count]) |
| | 766 | |
| | 767 | def join(self, connection, always_create=False, exclusions=(), |
| | 768 | promote=False, outer_if_first=False, nullable=False, reuse=None): |
| | 769 | """ |
| | 770 | Returns an alias for the join in 'connection', either reusing an |
| | 771 | existing alias for that join or creating a new one. 'connection' is a |
| | 772 | tuple (lhs, table, lhs_col, col) where 'lhs' is either an existing |
| | 773 | table alias or a table name. The join correspods to the SQL equivalent |
| | 774 | of:: |
| | 775 | |
| | 776 | lhs.lhs_col = table.col |
| | 777 | |
| | 778 | If 'always_create' is True and 'reuse' is None, a new alias is always |
| | 779 | created, regardless of whether one already exists or not. If |
| | 780 | 'always_create' is True and 'reuse' is a set, an alias in 'reuse' that |
| | 781 | matches the connection will be returned, if possible. If |
| | 782 | 'always_create' is False, the first existing alias that matches the |
| | 783 | 'connection' is returned, if any. Otherwise a new join is created. |
| | 784 | |
| | 785 | If 'exclusions' is specified, it is something satisfying the container |
| | 786 | protocol ("foo in exclusions" must work) and specifies a list of |
| | 787 | aliases that should not be returned, even if they satisfy the join. |
| | 788 | |
| | 789 | If 'promote' is True, the join type for the alias will be LOUTER (if |
| | 790 | the alias previously existed, the join type will be promoted from INNER |
| | 791 | to LOUTER, if necessary). |
| | 792 | |
| | 793 | If 'outer_if_first' is True and a new join is created, it will have the |
| | 794 | LOUTER join type. This is used when joining certain types of querysets |
| | 795 | and Q-objects together. |
| | 796 | |
| | 797 | If 'nullable' is True, the join can potentially involve NULL values and |
| | 798 | is a candidate for promotion (to "left outer") when combining querysets. |
| | 799 | """ |
| | 800 | lhs, table, lhs_col, col = connection |
| | 801 | if lhs in self.alias_map: |
| | 802 | lhs_table = self.alias_map[lhs][TABLE_NAME] |
| | 803 | else: |
| | 804 | lhs_table = lhs |
| | 805 | |
| | 806 | if reuse and always_create and table in self.table_map: |
| | 807 | # Convert the 'reuse' to case to be "exclude everything but the |
| | 808 | # reusable set, minus exclusions, for this table". |
| | 809 | exclusions = set(self.table_map[table]).difference(reuse).union(set(exclusions)) |
| | 810 | always_create = False |
| | 811 | t_ident = (lhs_table, table, lhs_col, col) |
| | 812 | if not always_create: |
| | 813 | for alias in self.join_map.get(t_ident, ()): |
| | 814 | if alias not in exclusions: |
| | 815 | if lhs_table and not self.alias_refcount[self.alias_map[alias][LHS_ALIAS]]: |
| | 816 | # The LHS of this join tuple is no longer part of the |
| | 817 | # query, so skip this possibility. |
| | 818 | continue |
| | 819 | if self.alias_map[alias][LHS_ALIAS] != lhs: |
| | 820 | continue |
| | 821 | self.ref_alias(alias) |
| | 822 | if promote: |
| | 823 | self.promote_alias(alias) |
| | 824 | return alias |
| | 825 | |
| | 826 | # No reuse is possible, so we need a new alias. |
| | 827 | alias, _ = self.table_alias(table, True) |
| | 828 | if not lhs: |
| | 829 | # Not all tables need to be joined to anything. No join type |
| | 830 | # means the later columns are ignored. |
| | 831 | join_type = None |
| | 832 | elif promote or outer_if_first: |
| | 833 | join_type = self.LOUTER |
| | 834 | else: |
| | 835 | join_type = self.INNER |
| | 836 | join = (table, alias, join_type, lhs, lhs_col, col, nullable) |
| | 837 | self.alias_map[alias] = join |
| | 838 | if t_ident in self.join_map: |
| | 839 | self.join_map[t_ident] += (alias,) |
| | 840 | else: |
| | 841 | self.join_map[t_ident] = (alias,) |
| | 842 | self.rev_join_map[alias] = t_ident |
| | 843 | return alias |
| | 844 | |
| | 845 | def setup_inherited_models(self): |
| | 846 | """ |
| | 847 | If the model that is the basis for this QuerySet inherits other models, |
| | 848 | we need to ensure that those other models have their tables included in |
| | 849 | the query. |
| | 850 | |
| | 851 | We do this as a separate step so that subclasses know which |
| | 852 | tables are going to be active in the query, without needing to compute |
| | 853 | all the select columns (this method is called from pre_sql_setup(), |
| | 854 | whereas column determination is a later part, and side-effect, of |
| | 855 | as_sql()). |
| | 856 | """ |
| | 857 | opts = self.model._meta |
| | 858 | root_alias = self.tables[0] |
| | 859 | seen = {None: root_alias} |
| | 860 | |
| | 861 | # Skip all proxy to the root proxied model |
| | 862 | proxied_model = get_proxied_model(opts) |
| | 863 | |
| | 864 | for field, model in opts.get_fields_with_model(): |
| | 865 | if model not in seen: |
| | 866 | if model is proxied_model: |
| | 867 | seen[model] = root_alias |
| | 868 | else: |
| | 869 | link_field = opts.get_ancestor_link(model) |
| | 870 | seen[model] = self.join((root_alias, model._meta.db_table, |
| | 871 | link_field.column, model._meta.pk.column)) |
| | 872 | self.included_inherited_models = seen |
| | 873 | |
| | 874 | def remove_inherited_models(self): |
| | 875 | """ |
| | 876 | Undoes the effects of setup_inherited_models(). Should be called |
| | 877 | whenever select columns (self.select) are set explicitly. |
| | 878 | """ |
| | 879 | for key, alias in self.included_inherited_models.items(): |
| | 880 | if key: |
| | 881 | self.unref_alias(alias) |
| | 882 | self.included_inherited_models = {} |
| | 883 | |
| | 884 | |
| | 885 | def add_aggregate(self, aggregate, model, alias, is_summary): |
| | 886 | """ |
| | 887 | Adds a single aggregate expression to the Query |
| | 888 | """ |
| | 889 | opts = model._meta |
| | 890 | field_list = aggregate.lookup.split(LOOKUP_SEP) |
| | 891 | if (len(field_list) == 1 and |
| | 892 | aggregate.lookup in self.aggregates.keys()): |
| | 893 | # Aggregate is over an annotation |
| | 894 | field_name = field_list[0] |
| | 895 | col = field_name |
| | 896 | source = self.aggregates[field_name] |
| | 897 | if not is_summary: |
| | 898 | raise FieldError("Cannot compute %s('%s'): '%s' is an aggregate" % ( |
| | 899 | aggregate.name, field_name, field_name)) |
| | 900 | elif ((len(field_list) > 1) or |
| | 901 | (field_list[0] not in [i.name for i in opts.fields]) or |
| | 902 | self.group_by is None or |
| | 903 | not is_summary): |
| | 904 | # If: |
| | 905 | # - the field descriptor has more than one part (foo__bar), or |
| | 906 | # - the field descriptor is referencing an m2m/m2o field, or |
| | 907 | # - this is a reference to a model field (possibly inherited), or |
| | 908 | # - this is an annotation over a model field |
| | 909 | # then we need to explore the joins that are required. |
| | 910 | |
| | 911 | field, source, opts, join_list, last, _ = self.setup_joins( |
| | 912 | field_list, opts, self.get_initial_alias(), False) |
| | 913 | |
| | 914 | # Process the join chain to see if it can be trimmed |
| | 915 | col, _, join_list = self.trim_joins(source, join_list, last, False) |
| | 916 | |
| | 917 | # If the aggregate references a model or field that requires a join, |
| | 918 | # those joins must be LEFT OUTER - empty join rows must be returned |
| | 919 | # in order for zeros to be returned for those aggregates. |
| | 920 | for column_alias in join_list: |
| | 921 | self.promote_alias(column_alias, unconditional=True) |
| | 922 | |
| | 923 | col = (join_list[-1], col) |
| | 924 | else: |
| | 925 | # The simplest cases. No joins required - |
| | 926 | # just reference the provided column alias. |
| | 927 | field_name = field_list[0] |
| | 928 | source = opts.get_field(field_name) |
| | 929 | col = field_name |
| | 930 | |
| | 931 | # Add the aggregate to the query |
| | 932 | aggregate.add_to_query(self, alias, col=col, source=source, is_summary=is_summary) |
| | 933 | |
| | 934 | def add_filter(self, filter_expr, connector=AND, negate=False, trim=False, |
| | 935 | can_reuse=None, process_extras=True): |
| | 936 | """ |
| | 937 | Add a single filter to the query. The 'filter_expr' is a pair: |
| | 938 | (filter_string, value). E.g. ('name__contains', 'fred') |
| | 939 | |
| | 940 | If 'negate' is True, this is an exclude() filter. It's important to |
| | 941 | note that this method does not negate anything in the where-clause |
| | 942 | object when inserting the filter constraints. This is because negated |
| | 943 | filters often require multiple calls to add_filter() and the negation |
| | 944 | should only happen once. So the caller is responsible for this (the |
| | 945 | caller will normally be add_q(), so that as an example). |
| | 946 | |
| | 947 | If 'trim' is True, we automatically trim the final join group (used |
| | 948 | internally when constructing nested queries). |
| | 949 | |
| | 950 | If 'can_reuse' is a set, we are processing a component of a |
| | 951 | multi-component filter (e.g. filter(Q1, Q2)). In this case, 'can_reuse' |
| | 952 | will be a set of table aliases that can be reused in this filter, even |
| | 953 | if we would otherwise force the creation of new aliases for a join |
| | 954 | (needed for nested Q-filters). The set is updated by this method. |
| | 955 | |
| | 956 | If 'process_extras' is set, any extra filters returned from the table |
| | 957 | joining process will be processed. This parameter is set to False |
| | 958 | during the processing of extra filters to avoid infinite recursion. |
| | 959 | """ |
| | 960 | arg, value = filter_expr |
| | 961 | parts = arg.split(LOOKUP_SEP) |
| | 962 | if not parts: |
| | 963 | raise FieldError("Cannot parse keyword query %r" % arg) |
| | 964 | |
| | 965 | # Work out the lookup type and remove it from 'parts', if necessary. |
| | 966 | if len(parts) == 1 or parts[-1] not in self.query_terms: |
| | 967 | lookup_type = 'exact' |
| | 968 | else: |
| | 969 | lookup_type = parts.pop() |
| | 970 | |
| | 971 | # By default, this is a WHERE clause. If an aggregate is referenced |
| | 972 | # in the value, the filter will be promoted to a HAVING |
| | 973 | having_clause = False |
| | 974 | |
| | 975 | # Interpret '__exact=None' as the sql 'is NULL'; otherwise, reject all |
| | 976 | # uses of None as a query value. |
| | 977 | if value is None: |
| | 978 | if lookup_type != 'exact': |
| | 979 | raise ValueError("Cannot use None as a query value") |
| | 980 | lookup_type = 'isnull' |
| | 981 | value = True |
| | 982 | elif callable(value): |
| | 983 | value = value() |
| | 984 | elif hasattr(value, 'evaluate'): |
| | 985 | # If value is a query expression, evaluate it |
| | 986 | value = SQLEvaluator(value, self) |
| | 987 | having_clause = value.contains_aggregate |
| | 988 | |
| | 989 | for alias, aggregate in self.aggregates.items(): |
| | 990 | if alias == parts[0]: |
| | 991 | entry = self.where_class() |
| | 992 | entry.add((aggregate, lookup_type, value), AND) |
| | 993 | if negate: |
| | 994 | entry.negate() |
| | 995 | self.having.add(entry, AND) |
| | 996 | return |
| | 997 | |
| | 998 | opts = self.get_meta() |
| | 999 | alias = self.get_initial_alias() |
| | 1000 | allow_many = trim or not negate |
| | 1001 | |
| | 1002 | try: |
| | 1003 | field, target, opts, join_list, last, extra_filters = self.setup_joins( |
| | 1004 | parts, opts, alias, True, allow_many, can_reuse=can_reuse, |
| | 1005 | negate=negate, process_extras=process_extras) |
| | 1006 | except MultiJoin, e: |
| | 1007 | self.split_exclude(filter_expr, LOOKUP_SEP.join(parts[:e.level]), |
| | 1008 | can_reuse) |
| | 1009 | return |
| | 1010 | |
| | 1011 | if (lookup_type == 'isnull' and value is True and not negate and |
| | 1012 | len(join_list) > 1): |
| | 1013 | # If the comparison is against NULL, we may need to use some left |
| | 1014 | # outer joins when creating the join chain. This is only done when |
| | 1015 | # needed, as it's less efficient at the database level. |
| | 1016 | self.promote_alias_chain(join_list) |
| | 1017 | |
| | 1018 | # Process the join list to see if we can remove any inner joins from |
| | 1019 | # the far end (fewer tables in a query is better). |
| | 1020 | col, alias, join_list = self.trim_joins(target, join_list, last, trim) |
| | 1021 | |
| | 1022 | if connector == OR: |
| | 1023 | # Some joins may need to be promoted when adding a new filter to a |
| | 1024 | # disjunction. We walk the list of new joins and where it diverges |
| | 1025 | # from any previous joins (ref count is 1 in the table list), we |
| | 1026 | # make the new additions (and any existing ones not used in the new |
| | 1027 | # join list) an outer join. |
| | 1028 | join_it = iter(join_list) |
| | 1029 | table_it = iter(self.tables) |
| | 1030 | join_it.next(), table_it.next() |
| | 1031 | table_promote = False |
| | 1032 | join_promote = False |
| | 1033 | for join in join_it: |
| | 1034 | table = table_it.next() |
| | 1035 | if join == table and self.alias_refcount[join] > 1: |
| | 1036 | continue |
| | 1037 | join_promote = self.promote_alias(join) |
| | 1038 | if table != join: |
| | 1039 | table_promote = self.promote_alias(table) |
| | 1040 | break |
| | 1041 | self.promote_alias_chain(join_it, join_promote) |
| | 1042 | self.promote_alias_chain(table_it, table_promote) |
| | 1043 | |
| | 1044 | |
| | 1045 | if having_clause: |
| | 1046 | self.having.add((Constraint(alias, col, field), lookup_type, value), |
| | 1047 | connector) |
| | 1048 | else: |
| | 1049 | self.where.add((Constraint(alias, col, field), lookup_type, value), |
| | 1050 | connector) |
| | 1051 | |
| | 1052 | if negate: |
| | 1053 | self.promote_alias_chain(join_list) |
| | 1054 | if lookup_type != 'isnull': |
| | 1055 | if len(join_list) > 1: |
| | 1056 | for alias in join_list: |
| | 1057 | if self.alias_map[alias][JOIN_TYPE] == self.LOUTER: |
| | 1058 | j_col = self.alias_map[alias][RHS_JOIN_COL] |
| | 1059 | entry = self.where_class() |
| | 1060 | entry.add((Constraint(alias, j_col, None), 'isnull', True), AND) |
| | 1061 | entry.negate() |
| | 1062 | self.where.add(entry, AND) |
| | 1063 | break |
| | 1064 | elif not (lookup_type == 'in' |
| | 1065 | and not hasattr(value, 'as_sql') |
| | 1066 | and not hasattr(value, '_as_sql') |
| | 1067 | and not value) and field.null: |
| | 1068 | # Leaky abstraction artifact: We have to specifically |
| | 1069 | # exclude the "foo__in=[]" case from this handling, because |
| | 1070 | # it's short-circuited in the Where class. |
| | 1071 | # We also need to handle the case where a subquery is provided |
| | 1072 | entry = self.where_class() |
| | 1073 | entry.add((Constraint(alias, col, None), 'isnull', True), AND) |
| | 1074 | entry.negate() |
| | 1075 | self.where.add(entry, AND) |
| | 1076 | |
| | 1077 | if can_reuse is not None: |
| | 1078 | can_reuse.update(join_list) |
| | 1079 | if process_extras: |
| | 1080 | for filter in extra_filters: |
| | 1081 | self.add_filter(filter, negate=negate, can_reuse=can_reuse, |
| | 1082 | process_extras=False) |
| | 1083 | |
| | 1084 | def add_q(self, q_object, used_aliases=None): |
| | 1085 | """ |
| | 1086 | Adds a Q-object to the current filter. |
| | 1087 | |
| | 1088 | Can also be used to add anything that has an 'add_to_query()' method. |
| | 1089 | """ |
| | 1090 | if used_aliases is None: |
| | 1091 | used_aliases = self.used_aliases |
| | 1092 | if hasattr(q_object, 'add_to_query'): |
| | 1093 | # Complex custom objects are responsible for adding themselves. |
| | 1094 | q_object.add_to_query(self, used_aliases) |
| | 1095 | else: |
| | 1096 | if self.where and q_object.connector != AND and len(q_object) > 1: |
| | 1097 | self.where.start_subtree(AND) |
| | 1098 | subtree = True |
| | 1099 | else: |
| | 1100 | subtree = False |
| | 1101 | connector = AND |
| | 1102 | for child in q_object.children: |
| | 1103 | if connector == OR: |
| | 1104 | refcounts_before = self.alias_refcount.copy() |
| | 1105 | self.where.start_subtree(connector) |
| | 1106 | if isinstance(child, Node): |
| | 1107 | self.add_q(child, used_aliases) |
| | 1108 | else: |
| | 1109 | self.add_filter(child, connector, q_object.negated, |
| | 1110 | can_reuse=used_aliases) |
| | 1111 | self.where.end_subtree() |
| | 1112 | if connector == OR: |
| | 1113 | # Aliases that were newly added or not used at all need to |
| | 1114 | # be promoted to outer joins if they are nullable relations. |
| | 1115 | # (they shouldn't turn the whole conditional into the empty |
| | 1116 | # set just because they don't match anything). |
| | 1117 | self.promote_unused_aliases(refcounts_before, used_aliases) |
| | 1118 | connector = q_object.connector |
| | 1119 | if q_object.negated: |
| | 1120 | self.where.negate() |
| | 1121 | if subtree: |
| | 1122 | self.where.end_subtree() |
| | 1123 | if self.filter_is_sticky: |
| | 1124 | self.used_aliases = used_aliases |
| | 1125 | |
| | 1126 | def setup_joins(self, names, opts, alias, dupe_multis, allow_many=True, |
| | 1127 | allow_explicit_fk=False, can_reuse=None, negate=False, |
| | 1128 | process_extras=True): |
| | 1129 | """ |
| | 1130 | Compute the necessary table joins for the passage through the fields |
| | 1131 | given in 'names'. 'opts' is the Options class for the current model |
| | 1132 | (which gives the table we are joining to), 'alias' is the alias for the |
| | 1133 | table we are joining to. If dupe_multis is True, any many-to-many or |
| | 1134 | many-to-one joins will always create a new alias (necessary for |
| | 1135 | disjunctive filters). If can_reuse is not None, it's a list of aliases |
| | 1136 | that can be reused in these joins (nothing else can be reused in this |
| | 1137 | case). Finally, 'negate' is used in the same sense as for add_filter() |
| | 1138 | -- it indicates an exclude() filter, or something similar. It is only |
| | 1139 | passed in here so that it can be passed to a field's extra_filter() for |
| | 1140 | customised behaviour. |
| | 1141 | |
| | 1142 | Returns the final field involved in the join, the target database |
| | 1143 | column (used for any 'where' constraint), the final 'opts' value and the |
| | 1144 | list of tables joined. |
| | 1145 | """ |
| | 1146 | joins = [alias] |
| | 1147 | last = [0] |
| | 1148 | dupe_set = set() |
| | 1149 | exclusions = set() |
| | 1150 | extra_filters = [] |
| | 1151 | for pos, name in enumerate(names): |
| | 1152 | try: |
| | 1153 | exclusions.add(int_alias) |
| | 1154 | except NameError: |
| | 1155 | pass |
| | 1156 | exclusions.add(alias) |
| | 1157 | last.append(len(joins)) |
| | 1158 | if name == 'pk': |
| | 1159 | name = opts.pk.name |
| | 1160 | try: |
| | 1161 | field, model, direct, m2m = opts.get_field_by_name(name) |
| | 1162 | except FieldDoesNotExist: |
| | 1163 | for f in opts.fields: |
| | 1164 | if allow_explicit_fk and name == f.attname: |
| | 1165 | # XXX: A hack to allow foo_id to work in values() for |
| | 1166 | # backwards compatibility purposes. If we dropped that |
| | 1167 | # feature, this could be removed. |
| | 1168 | field, model, direct, m2m = opts.get_field_by_name(f.name) |
| | 1169 | break |
| | 1170 | else: |
| | 1171 | names = opts.get_all_field_names() + self.aggregate_select.keys() |
| | 1172 | raise FieldError("Cannot resolve keyword %r into field. " |
| | 1173 | "Choices are: %s" % (name, ", ".join(names))) |
| | 1174 | |
| | 1175 | if not allow_many and (m2m or not direct): |
| | 1176 | for alias in joins: |
| | 1177 | self.unref_alias(alias) |
| | 1178 | raise MultiJoin(pos + 1) |
| | 1179 | if model: |
| | 1180 | # The field lives on a base class of the current model. |
| | 1181 | # Skip the chain of proxy to the concrete proxied model |
| | 1182 | proxied_model = get_proxied_model(opts) |
| | 1183 | |
| | 1184 | for int_model in opts.get_base_chain(model): |
| | 1185 | if int_model is proxied_model: |
| | 1186 | opts = int_model._meta |
| | 1187 | else: |
| | 1188 | lhs_col = opts.parents[int_model].column |
| | 1189 | dedupe = lhs_col in opts.duplicate_targets |
| | 1190 | if dedupe: |
| | 1191 | exclusions.update(self.dupe_avoidance.get( |
| | 1192 | (id(opts), lhs_col), ())) |
| | 1193 | dupe_set.add((opts, lhs_col)) |
| | 1194 | opts = int_model._meta |
| | 1195 | alias = self.join((alias, opts.db_table, lhs_col, |
| | 1196 | opts.pk.column), exclusions=exclusions) |
| | 1197 | joins.append(alias) |
| | 1198 | exclusions.add(alias) |
| | 1199 | for (dupe_opts, dupe_col) in dupe_set: |
| | 1200 | self.update_dupe_avoidance(dupe_opts, dupe_col, |
| | 1201 | alias) |
| | 1202 | cached_data = opts._join_cache.get(name) |
| | 1203 | orig_opts = opts |
| | 1204 | dupe_col = direct and field.column or field.field.column |
| | 1205 | dedupe = dupe_col in opts.duplicate_targets |
| | 1206 | if dupe_set or dedupe: |
| | 1207 | if dedupe: |
| | 1208 | dupe_set.add((opts, dupe_col)) |
| | 1209 | exclusions.update(self.dupe_avoidance.get((id(opts), dupe_col), |
| | 1210 | ())) |
| | 1211 | |
| | 1212 | if process_extras and hasattr(field, 'extra_filters'): |
| | 1213 | extra_filters.extend(field.extra_filters(names, pos, negate)) |
| | 1214 | if direct: |
| | 1215 | if m2m: |
| | 1216 | # Many-to-many field defined on the current model. |
| | 1217 | if cached_data: |
| | 1218 | (table1, from_col1, to_col1, table2, from_col2, |
| | 1219 | to_col2, opts, target) = cached_data |
| | 1220 | else: |
| | 1221 | table1 = field.m2m_db_table() |
| | 1222 | from_col1 = opts.pk.column |
| | 1223 | to_col1 = field.m2m_column_name() |
| | 1224 | opts = field.rel.to._meta |
| | 1225 | table2 = opts.db_table |
| | 1226 | from_col2 = field.m2m_reverse_name() |
| | 1227 | to_col2 = opts.pk.column |
| | 1228 | target = opts.pk |
| | 1229 | orig_opts._join_cache[name] = (table1, from_col1, |
| | 1230 | to_col1, table2, from_col2, to_col2, opts, |
| | 1231 | target) |
| | 1232 | |
| | 1233 | int_alias = self.join((alias, table1, from_col1, to_col1), |
| | 1234 | dupe_multis, exclusions, nullable=True, |
| | 1235 | reuse=can_reuse) |
| | 1236 | if int_alias == table2 and from_col2 == to_col2: |
| | 1237 | joins.append(int_alias) |
| | 1238 | alias = int_alias |
| | 1239 | else: |
| | 1240 | alias = self.join( |
| | 1241 | (int_alias, table2, from_col2, to_col2), |
| | 1242 | dupe_multis, exclusions, nullable=True, |
| | 1243 | reuse=can_reuse) |
| | 1244 | joins.extend([int_alias, alias]) |
| | 1245 | elif field.rel: |
| | 1246 | # One-to-one or many-to-one field |
| | 1247 | if cached_data: |
| | 1248 | (table, from_col, to_col, opts, target) = cached_data |
| | 1249 | else: |
| | 1250 | opts = field.rel.to._meta |
| | 1251 | target = field.rel.get_related_field() |
| | 1252 | table = opts.db_table |
| | 1253 | from_col = field.column |
| | 1254 | to_col = target.column |
| | 1255 | orig_opts._join_cache[name] = (table, from_col, to_col, |
| | 1256 | opts, target) |
| | 1257 | |
| | 1258 | alias = self.join((alias, table, from_col, to_col), |
| | 1259 | exclusions=exclusions, nullable=field.null) |
| | 1260 | joins.append(alias) |
| | 1261 | else: |
| | 1262 | # Non-relation fields. |
| | 1263 | target = field |
| | 1264 | break |
| | 1265 | else: |
| | 1266 | orig_field = field |
| | 1267 | field = field.field |
| | 1268 | if m2m: |
| | 1269 | # Many-to-many field defined on the target model. |
| | 1270 | if cached_data: |
| | 1271 | (table1, from_col1, to_col1, table2, from_col2, |
| | 1272 | to_col2, opts, target) = cached_data |
| | 1273 | else: |
| | 1274 | table1 = field.m2m_db_table() |
| | 1275 | from_col1 = opts.pk.column |
| | 1276 | to_col1 = field.m2m_reverse_name() |
| | 1277 | opts = orig_field.opts |
| | 1278 | table2 = opts.db_table |
| | 1279 | from_col2 = field.m2m_column_name() |
| | 1280 | to_col2 = opts.pk.column |
| | 1281 | target = opts.pk |
| | 1282 | orig_opts._join_cache[name] = (table1, from_col1, |
| | 1283 | to_col1, table2, from_col2, to_col2, opts, |
| | 1284 | target) |
| | 1285 | |
| | 1286 | int_alias = self.join((alias, table1, from_col1, to_col1), |
| | 1287 | dupe_multis, exclusions, nullable=True, |
| | 1288 | reuse=can_reuse) |
| | 1289 | alias = self.join((int_alias, table2, from_col2, to_col2), |
| | 1290 | dupe_multis, exclusions, nullable=True, |
| | 1291 | reuse=can_reuse) |
| | 1292 | joins.extend([int_alias, alias]) |
| | 1293 | else: |
| | 1294 | # One-to-many field (ForeignKey defined on the target model) |
| | 1295 | if cached_data: |
| | 1296 | (table, from_col, to_col, opts, target) = cached_data |
| | 1297 | else: |
| | 1298 | local_field = opts.get_field_by_name( |
| | 1299 | field.rel.field_name)[0] |
| | 1300 | opts = orig_field.opts |
| | 1301 | table = opts.db_table |
| | 1302 | from_col = local_field.column |
| | 1303 | to_col = field.column |
| | 1304 | target = opts.pk |
| | 1305 | orig_opts._join_cache[name] = (table, from_col, to_col, |
| | 1306 | opts, target) |
| | 1307 | |
| | 1308 | alias = self.join((alias, table, from_col, to_col), |
| | 1309 | dupe_multis, exclusions, nullable=True, |
| | 1310 | reuse=can_reuse) |
| | 1311 | joins.append(alias) |
| | 1312 | |
| | 1313 | for (dupe_opts, dupe_col) in dupe_set: |
| | 1314 | try: |
| | 1315 | self.update_dupe_avoidance(dupe_opts, dupe_col, int_alias) |
| | 1316 | except NameError: |
| | 1317 | self.update_dupe_avoidance(dupe_opts, dupe_col, alias) |
| | 1318 | |
| | 1319 | if pos != len(names) - 1: |
| | 1320 | if pos == len(names) - 2: |
| | 1321 | raise FieldError("Join on field %r not permitted. Did you misspell %r for the lookup type?" % (name, names[pos + 1])) |
| | 1322 | else: |
| | 1323 | raise FieldError("Join on field %r not permitted." % name) |
| | 1324 | |
| | 1325 | return field, target, opts, joins, last, extra_filters |
| | 1326 | |
| | 1327 | def trim_joins(self, target, join_list, last, trim): |
| | 1328 | """ |
| | 1329 | Sometimes joins at the end of a multi-table sequence can be trimmed. If |
| | 1330 | the final join is against the same column as we are comparing against, |
| | 1331 | and is an inner join, we can go back one step in a join chain and |
| | 1332 | compare against the LHS of the join instead (and then repeat the |
| | 1333 | optimization). The result, potentially, involves less table joins. |
| | 1334 | |
| | 1335 | The 'target' parameter is the final field being joined to, 'join_list' |
| | 1336 | is the full list of join aliases. |
| | 1337 | |
| | 1338 | The 'last' list contains offsets into 'join_list', corresponding to |
| | 1339 | each component of the filter. Many-to-many relations, for example, add |
| | 1340 | two tables to the join list and we want to deal with both tables the |
| | 1341 | same way, so 'last' has an entry for the first of the two tables and |
| | 1342 | then the table immediately after the second table, in that case. |
| | 1343 | |
| | 1344 | The 'trim' parameter forces the final piece of the join list to be |
| | 1345 | trimmed before anything. See the documentation of add_filter() for |
| | 1346 | details about this. |
| | 1347 | |
| | 1348 | Returns the final active column and table alias and the new active |
| | 1349 | join_list. |
| | 1350 | """ |
| | 1351 | final = len(join_list) |
| | 1352 | penultimate = last.pop() |
| | 1353 | if penultimate == final: |
| | 1354 | penultimate = last.pop() |
| | 1355 | if trim and len(join_list) > 1: |
| | 1356 | extra = join_list[penultimate:] |
| | 1357 | join_list = join_list[:penultimate] |
| | 1358 | final = penultimate |
| | 1359 | penultimate = last.pop() |
| | 1360 | col = self.alias_map[extra[0]][LHS_JOIN_COL] |
| | 1361 | for alias in extra: |
| | 1362 | self.unref_alias(alias) |
| | 1363 | else: |
| | 1364 | col = target.column |
| | 1365 | alias = join_list[-1] |
| | 1366 | while final > 1: |
| | 1367 | join = self.alias_map[alias] |
| | 1368 | if col != join[RHS_JOIN_COL] or join[JOIN_TYPE] != self.INNER: |
| | 1369 | break |
| | 1370 | self.unref_alias(alias) |
| | 1371 | alias = join[LHS_ALIAS] |
| | 1372 | col = join[LHS_JOIN_COL] |
| | 1373 | join_list = join_list[:-1] |
| | 1374 | final -= 1 |
| | 1375 | if final == penultimate: |
| | 1376 | penultimate = last.pop() |
| | 1377 | return col, alias, join_list |
| | 1378 | |
| | 1379 | def update_dupe_avoidance(self, opts, col, alias): |
| | 1380 | """ |
| | 1381 | For a column that is one of multiple pointing to the same table, update |
| | 1382 | the internal data structures to note that this alias shouldn't be used |
| | 1383 | for those other columns. |
| | 1384 | """ |
| | 1385 | ident = id(opts) |
| | 1386 | for name in opts.duplicate_targets[col]: |
| | 1387 | try: |
| | 1388 | self.dupe_avoidance[ident, name].add(alias) |
| | 1389 | except KeyError: |
| | 1390 | self.dupe_avoidance[ident, name] = set([alias]) |
| | 1391 | |
| | 1392 | def split_exclude(self, filter_expr, prefix, can_reuse): |
| | 1393 | """ |
| | 1394 | When doing an exclude against any kind of N-to-many relation, we need |
| | 1395 | to use a subquery. This method constructs the nested query, given the |
| | 1396 | original exclude filter (filter_expr) and the portion up to the first |
| | 1397 | N-to-many relation field. |
| | 1398 | """ |
| | 1399 | query = Query(self.model) |
| | 1400 | query.add_filter(filter_expr, can_reuse=can_reuse) |
| | 1401 | query.bump_prefix() |
| | 1402 | query.clear_ordering(True) |
| | 1403 | query.set_start(prefix) |
| | 1404 | self.add_filter(('%s__in' % prefix, query), negate=True, trim=True, |
| | 1405 | can_reuse=can_reuse) |
| | 1406 | |
| | 1407 | # If there's more than one join in the inner query (before any initial |
| | 1408 | # bits were trimmed -- which means the last active table is more than |
| | 1409 | # two places into the alias list), we need to also handle the |
| | 1410 | # possibility that the earlier joins don't match anything by adding a |
| | 1411 | # comparison to NULL (e.g. in |
| | 1412 | # Tag.objects.exclude(parent__parent__name='t1'), a tag with no parent |
| | 1413 | # would otherwise be overlooked). |
| | 1414 | active_positions = [pos for (pos, count) in |
| | 1415 | enumerate(query.alias_refcount.itervalues()) if count] |
| | 1416 | if active_positions[-1] > 1: |
| | 1417 | self.add_filter(('%s__isnull' % prefix, False), negate=True, |
| | 1418 | trim=True, can_reuse=can_reuse) |
| | 1419 | |
| | 1420 | def set_limits(self, low=None, high=None): |
| | 1421 | """ |
| | 1422 | Adjusts the limits on the rows retrieved. We use low/high to set these, |
| | 1423 | as it makes it more Pythonic to read and write. When the SQL query is |
| | 1424 | created, they are converted to the appropriate offset and limit values. |
| | 1425 | |
| | 1426 | Any limits passed in here are applied relative to the existing |
| | 1427 | constraints. So low is added to the current low value and both will be |
| | 1428 | clamped to any existing high value. |
| | 1429 | """ |
| | 1430 | if high is not None: |
| | 1431 | if self.high_mark is not None: |
| | 1432 | self.high_mark = min(self.high_mark, self.low_mark + high) |
| | 1433 | else: |
| | 1434 | self.high_mark = self.low_mark + high |
| | 1435 | if low is not None: |
| | 1436 | if self.high_mark is not None: |
| | 1437 | self.low_mark = min(self.high_mark, self.low_mark + low) |
| | 1438 | else: |
| | 1439 | self.low_mark = self.low_mark + low |
| | 1440 | |
| | 1441 | def clear_limits(self): |
| | 1442 | """ |
| | 1443 | Clears any existing limits. |
| | 1444 | """ |
| | 1445 | self.low_mark, self.high_mark = 0, None |
| | 1446 | |
| | 1447 | def can_filter(self): |
| | 1448 | """ |
| | 1449 | Returns True if adding filters to this instance is still possible. |
| | 1450 | |
| | 1451 | Typically, this means no limits or offsets have been put on the results. |
| | 1452 | """ |
| | 1453 | return not self.low_mark and self.high_mark is None |
| | 1454 | |
| | 1455 | def clear_select_fields(self): |
| | 1456 | """ |
| | 1457 | Clears the list of fields to select (but not extra_select columns). |
| | 1458 | Some queryset types completely replace any existing list of select |
| | 1459 | columns. |
| | 1460 | """ |
| | 1461 | self.select = [] |
| | 1462 | self.select_fields = [] |
| | 1463 | |
| | 1464 | def add_fields(self, field_names, allow_m2m=True): |
| | 1465 | """ |
| | 1466 | Adds the given (model) fields to the select set. The field names are |
| | 1467 | added in the order specified. |
| | 1468 | """ |
| | 1469 | alias = self.get_initial_alias() |
| | 1470 | opts = self.get_meta() |
| | 1471 | |
| | 1472 | try: |
| | 1473 | for name in field_names: |
| | 1474 | field, target, u2, joins, u3, u4 = self.setup_joins( |
| | 1475 | name.split(LOOKUP_SEP), opts, alias, False, allow_m2m, |
| | 1476 | True) |
| | 1477 | final_alias = joins[-1] |
| | 1478 | col = target.column |
| | 1479 | if len(joins) > 1: |
| | 1480 | join = self.alias_map[final_alias] |
| | 1481 | if col == join[RHS_JOIN_COL]: |
| | 1482 | self.unref_alias(final_alias) |
| | 1483 | final_alias = join[LHS_ALIAS] |
| | 1484 | col = join[LHS_JOIN_COL] |
| | 1485 | joins = joins[:-1] |
| | 1486 | self.promote_alias_chain(joins[1:]) |
| | 1487 | self.select.append((final_alias, col)) |
| | 1488 | self.select_fields.append(field) |
| | 1489 | except MultiJoin: |
| | 1490 | raise FieldError("Invalid field name: '%s'" % name) |
| | 1491 | except FieldError: |
| | 1492 | names = opts.get_all_field_names() + self.extra.keys() + self.aggregate_select.keys() |
| | 1493 | names.sort() |
| | 1494 | raise FieldError("Cannot resolve keyword %r into field. " |
| | 1495 | "Choices are: %s" % (name, ", ".join(names))) |
| | 1496 | self.remove_inherited_models() |
| | 1497 | |
| | 1498 | def add_ordering(self, *ordering): |
| | 1499 | """ |
| | 1500 | Adds items from the 'ordering' sequence to the query's "order by" |
| | 1501 | clause. These items are either field names (not column names) -- |
| | 1502 | possibly with a direction prefix ('-' or '?') -- or ordinals, |
| | 1503 | corresponding to column positions in the 'select' list. |
| | 1504 | |
| | 1505 | If 'ordering' is empty, all ordering is cleared from the query. |
| | 1506 | """ |
| | 1507 | errors = [] |
| | 1508 | for item in ordering: |
| | 1509 | if not ORDER_PATTERN.match(item): |
| | 1510 | errors.append(item) |
| | 1511 | if errors: |
| | 1512 | raise FieldError('Invalid order_by arguments: %s' % errors) |
| | 1513 | if ordering: |
| | 1514 | self.order_by.extend(ordering) |
| | 1515 | else: |
| | 1516 | self.default_ordering = False |
| | 1517 | |
| | 1518 | def clear_ordering(self, force_empty=False): |
| | 1519 | """ |
| | 1520 | Removes any ordering settings. If 'force_empty' is True, there will be |
| | 1521 | no ordering in the resulting query (not even the model's default). |
| | 1522 | """ |
| | 1523 | self.order_by = [] |
| | 1524 | self.extra_order_by = () |
| | 1525 | if force_empty: |
| | 1526 | self.default_ordering = False |
| | 1527 | |
| | 1528 | def set_group_by(self): |
| | 1529 | """ |
| | 1530 | Expands the GROUP BY clause required by the query. |
| | 1531 | |
| | 1532 | This will usually be the set of all non-aggregate fields in the |
| | 1533 | return data. If the database backend supports grouping by the |
| | 1534 | primary key, and the query would be equivalent, the optimization |
| | 1535 | will be made automatically. |
| | 1536 | """ |
| | 1537 | self.group_by = [] |
| | 1538 | |
| | 1539 | for sel in self.select: |
| | 1540 | self.group_by.append(sel) |
| | 1541 | |
| | 1542 | def add_count_column(self): |
| | 1543 | """ |
| | 1544 | Converts the query to do count(...) or count(distinct(pk)) in order to |
| | 1545 | get its size. |
| | 1546 | """ |
| | 1547 | if not self.distinct: |
| | 1548 | if not self.select: |
| | 1549 | count = self.aggregates_module.Count('*', is_summary=True) |
| | 1550 | else: |
| | 1551 | assert len(self.select) == 1, \ |
| | 1552 | "Cannot add count col with multiple cols in 'select': %r" % self.select |
| | 1553 | count = self.aggregates_module.Count(self.select[0]) |
| | 1554 | else: |
| | 1555 | opts = self.model._meta |
| | 1556 | if not self.select: |
| | 1557 | count = self.aggregates_module.Count((self.join((None, opts.db_table, None, None)), opts.pk.column), |
| | 1558 | is_summary=True, distinct=True) |
| | 1559 | else: |
| | 1560 | # Because of SQL portability issues, multi-column, distinct |
| | 1561 | # counts need a sub-query -- see get_count() for details. |
| | 1562 | assert len(self.select) == 1, \ |
| | 1563 | "Cannot add count col with multiple cols in 'select'." |
| | 1564 | |
| | 1565 | count = self.aggregates_module.Count(self.select[0], distinct=True) |
| | 1566 | # Distinct handling is done in Count(), so don't do it at this |
| | 1567 | # level. |
| | 1568 | self.distinct = False |
| | 1569 | |
| | 1570 | # Set only aggregate to be the count column. |
| | 1571 | # Clear out the select cache to reflect the new unmasked aggregates. |
| | 1572 | self.aggregates = {None: count} |
| | 1573 | self.set_aggregate_mask(None) |
| | 1574 | self.group_by = None |
| | 1575 | |
| | 1576 | def add_select_related(self, fields): |
| | 1577 | """ |
| | 1578 | Sets up the select_related data structure so that we only select |
| | 1579 | certain related models (as opposed to all models, when |
| | 1580 | self.select_related=True). |
| | 1581 | """ |
| | 1582 | field_dict = {} |
| | 1583 | for field in fields: |
| | 1584 | d = field_dict |
| | 1585 | for part in field.split(LOOKUP_SEP): |
| | 1586 | d = d.setdefault(part, {}) |
| | 1587 | self.select_related = field_dict |
| | 1588 | self.related_select_cols = [] |
| | 1589 | self.related_select_fields = [] |
| | 1590 | |
| | 1591 | def add_extra(self, select, select_params, where, params, tables, order_by): |
| | 1592 | """ |
| | 1593 | Adds data to the various extra_* attributes for user-created additions |
| | 1594 | to the query. |
| | 1595 | """ |
| | 1596 | if select: |
| | 1597 | # We need to pair any placeholder markers in the 'select' |
| | 1598 | # dictionary with their parameters in 'select_params' so that |
| | 1599 | # subsequent updates to the select dictionary also adjust the |
| | 1600 | # parameters appropriately. |
| | 1601 | select_pairs = SortedDict() |
| | 1602 | if select_params: |
| | 1603 | param_iter = iter(select_params) |
| | 1604 | else: |
| | 1605 | param_iter = iter([]) |
| | 1606 | for name, entry in select.items(): |
| | 1607 | entry = force_unicode(entry) |
| | 1608 | entry_params = [] |
| | 1609 | pos = entry.find("%s") |
| | 1610 | while pos != -1: |
| | 1611 | entry_params.append(param_iter.next()) |
| | 1612 | pos = entry.find("%s", pos + 2) |
| | 1613 | select_pairs[name] = (entry, entry_params) |
| | 1614 | # This is order preserving, since self.extra_select is a SortedDict. |
| | 1615 | self.extra.update(select_pairs) |
| | 1616 | if where or params: |
| | 1617 | self.where.add(ExtraWhere(where, params), AND) |
| | 1618 | if tables: |
| | 1619 | self.extra_tables += tuple(tables) |
| | 1620 | if order_by: |
| | 1621 | self.extra_order_by = order_by |
| | 1622 | |
| | 1623 | def clear_deferred_loading(self): |
| | 1624 | """ |
| | 1625 | Remove any fields from the deferred loading set. |
| | 1626 | """ |
| | 1627 | self.deferred_loading = (set(), True) |
| | 1628 | |
| | 1629 | def add_deferred_loading(self, field_names): |
| | 1630 | """ |
| | 1631 | Add the given list of model field names to the set of fields to |
| | 1632 | exclude from loading from the database when automatic column selection |
| | 1633 | is done. The new field names are added to any existing field names that |
| | 1634 | are deferred (or removed from any existing field names that are marked |
| | 1635 | as the only ones for immediate loading). |
| | 1636 | """ |
| | 1637 | # Fields on related models are stored in the literal double-underscore |
| | 1638 | # format, so that we can use a set datastructure. We do the foo__bar |
| | 1639 | # splitting and handling when computing the SQL colum names (as part of |
| | 1640 | # get_columns()). |
| | 1641 | existing, defer = self.deferred_loading |
| | 1642 | if defer: |
| | 1643 | # Add to existing deferred names. |
| | 1644 | self.deferred_loading = existing.union(field_names), True |
| | 1645 | else: |
| | 1646 | # Remove names from the set of any existing "immediate load" names. |
| | 1647 | self.deferred_loading = existing.difference(field_names), False |
| | 1648 | |
| | 1649 | def add_immediate_loading(self, field_names): |
| | 1650 | """ |
| | 1651 | Add the given list of model field names to the set of fields to |
| | 1652 | retrieve when the SQL is executed ("immediate loading" fields). The |
| | 1653 | field names replace any existing immediate loading field names. If |
| | 1654 | there are field names already specified for deferred loading, those |
| | 1655 | names are removed from the new field_names before storing the new names |
| | 1656 | for immediate loading. (That is, immediate loading overrides any |
| | 1657 | existing immediate values, but respects existing deferrals.) |
| | 1658 | """ |
| | 1659 | existing, defer = self.deferred_loading |
| | 1660 | if defer: |
| | 1661 | # Remove any existing deferred names from the current set before |
| | 1662 | # setting the new names. |
| | 1663 | self.deferred_loading = set(field_names).difference(existing), False |
| | 1664 | else: |
| | 1665 | # Replace any existing "immediate load" field names. |
| | 1666 | self.deferred_loading = set(field_names), False |
| | 1667 | |
| | 1668 | def get_loaded_field_names(self): |
| | 1669 | """ |
| | 1670 | If any fields are marked to be deferred, returns a dictionary mapping |
| | 1671 | models to a set of names in those fields that will be loaded. If a |
| | 1672 | model is not in the returned dictionary, none of it's fields are |
| | 1673 | deferred. |
| | 1674 | |
| | 1675 | If no fields are marked for deferral, returns an empty dictionary. |
| | 1676 | """ |
| | 1677 | collection = {} |
| | 1678 | self.deferred_to_data(collection, self.get_loaded_field_names_cb) |
| | 1679 | return collection |
| | 1680 | |
| | 1681 | def get_loaded_field_names_cb(self, target, model, fields): |
| | 1682 | """ |
| | 1683 | Callback used by get_deferred_field_names(). |
| | 1684 | """ |
| | 1685 | target[model] = set([f.name for f in fields]) |
| | 1686 | |
| | 1687 | def set_aggregate_mask(self, names): |
| | 1688 | "Set the mask of aggregates that will actually be returned by the SELECT" |
| | 1689 | if names is None: |
| | 1690 | self.aggregate_select_mask = None |
| | 1691 | else: |
| | 1692 | self.aggregate_select_mask = set(names) |
| | 1693 | self._aggregate_select_cache = None |
| | 1694 | |
| | 1695 | def set_extra_mask(self, names): |
| | 1696 | """ |
| | 1697 | Set the mask of extra select items that will be returned by SELECT, |
| | 1698 | we don't actually remove them from the Query since they might be used |
| | 1699 | later |
| | 1700 | """ |
| | 1701 | if names is None: |
| | 1702 | self.extra_select_mask = None |
| | 1703 | else: |
| | 1704 | self.extra_select_mask = set(names) |
| | 1705 | self._extra_select_cache = None |
| | 1706 | |
| | 1707 | def _aggregate_select(self): |
| | 1708 | """The SortedDict of aggregate columns that are not masked, and should |
| | 1709 | be used in the SELECT clause. |
| | 1710 | |
| | 1711 | This result is cached for optimization purposes. |
| | 1712 | """ |
| | 1713 | if self._aggregate_select_cache is not None: |
| | 1714 | return self._aggregate_select_cache |
| | 1715 | elif self.aggregate_select_mask is not None: |
| | 1716 | self._aggregate_select_cache = SortedDict([ |
| | 1717 | (k,v) for k,v in self.aggregates.items() |
| | 1718 | if k in self.aggregate_select_mask |
| | 1719 | ]) |
| | 1720 | return self._aggregate_select_cache |
| | 1721 | else: |
| | 1722 | return self.aggregates |
| | 1723 | aggregate_select = property(_aggregate_select) |
| | 1724 | |
| | 1725 | def _extra_select(self): |
| | 1726 | if self._extra_select_cache is not None: |
| | 1727 | return self._extra_select_cache |
| | 1728 | elif self.extra_select_mask is not None: |
| | 1729 | self._extra_select_cache = SortedDict([ |
| | 1730 | (k,v) for k,v in self.extra.items() |
| | 1731 | if k in self.extra_select_mask |
| | 1732 | ]) |
| | 1733 | return self._extra_select_cache |
| | 1734 | else: |
| | 1735 | return self.extra |
| | 1736 | extra_select = property(_extra_select) |
| | 1737 | |
| | 1738 | def set_start(self, start): |
| | 1739 | """ |
| | 1740 | Sets the table from which to start joining. The start position is |
| | 1741 | specified by the related attribute from the base model. This will |
| | 1742 | automatically set to the select column to be the column linked from the |
| | 1743 | previous table. |
| | 1744 | |
| | 1745 | This method is primarily for internal use and the error checking isn't |
| | 1746 | as friendly as add_filter(). Mostly useful for querying directly |
| | 1747 | against the join table of many-to-many relation in a subquery. |
| | 1748 | """ |
| | 1749 | opts = self.model._meta |
| | 1750 | alias = self.get_initial_alias() |
| | 1751 | field, col, opts, joins, last, extra = self.setup_joins( |
| | 1752 | start.split(LOOKUP_SEP), opts, alias, False) |
| | 1753 | select_col = self.alias_map[joins[1]][LHS_JOIN_COL] |
| | 1754 | select_alias = alias |
| | 1755 | |
| | 1756 | # The call to setup_joins added an extra reference to everything in |
| | 1757 | # joins. Reverse that. |
| | 1758 | for alias in joins: |
| | 1759 | self.unref_alias(alias) |
| | 1760 | |
| | 1761 | # We might be able to trim some joins from the front of this query, |
| | 1762 | # providing that we only traverse "always equal" connections (i.e. rhs |
| | 1763 | # is *always* the same value as lhs). |
| | 1764 | for alias in joins[1:]: |
| | 1765 | join_info = self.alias_map[alias] |
| | 1766 | if (join_info[LHS_JOIN_COL] != select_col |
| | 1767 | or join_info[JOIN_TYPE] != self.INNER): |
| | 1768 | break |
| | 1769 | self.unref_alias(select_alias) |
| | 1770 | select_alias = join_info[RHS_ALIAS] |
| | 1771 | select_col = join_info[RHS_JOIN_COL] |
| | 1772 | self.select = [(select_alias, select_col)] |
| | 1773 | self.remove_inherited_models() |
| | 1774 | |
| | 1775 | |
| | 1776 | def get_order_dir(field, default='ASC'): |
| | 1777 | """ |
| | 1778 | Returns the field name and direction for an order specification. For |
| | 1779 | example, '-foo' is returned as ('foo', 'DESC'). |
| | 1780 | |
| | 1781 | The 'default' param is used to indicate which way no prefix (or a '+' |
| | 1782 | prefix) should sort. The '-' prefix always sorts the opposite way. |
| | 1783 | """ |
| | 1784 | dirn = ORDER_DIR[default] |
| | 1785 | if field[0] == '-': |
| | 1786 | return field[1:], dirn[1] |
| | 1787 | return field, dirn[0] |
| | 1788 | |
| | 1789 | |
| | 1790 | def setup_join_cache(sender, **kwargs): |
| | 1791 | """ |
| | 1792 | The information needed to join between model fields is something that is |
| | 1793 | invariant over the life of the model, so we cache it in the model's Options |
| | 1794 | class, rather than recomputing it all the time. |
| | 1795 | |
| | 1796 | This method initialises the (empty) cache when the model is created. |
| | 1797 | """ |
| | 1798 | sender._meta._join_cache = {} |
| | 1799 | |
| | 1800 | signals.class_prepared.connect(setup_join_cache) |
| | 1801 | |
| | 1802 | def add_to_dict(data, key, value): |
| | 1803 | """ |
| | 1804 | A helper function to add "value" to the set of values for "key", whether or |
| | 1805 | not "key" already exists. |
| | 1806 | """ |
| | 1807 | if key in data: |
| | 1808 | data[key].add(value) |
| | 1809 | else: |
| | 1810 | data[key] = set([value]) |
| | 1811 | |
| | 1812 | def get_proxied_model(opts): |
| | 1813 | int_opts = opts |
| | 1814 | proxied_model = None |
| | 1815 | while int_opts.proxy: |
| | 1816 | proxied_model = int_opts.proxy_for_model |
| | 1817 | int_opts = proxied_model._meta |
| | 1818 | return proxied_model |