query.py

Revision 2803, 36.5 kB (checked in by adrian, 2 years ago)
magic-removal: Fixed #1685 -- order_by no longer breaks when using a custom 'select'. Thanks, feiyu.xie

Line
1	from django.db import backend, connection, transaction
2	from django.db.models.fields import DateField, FieldDoesNotExist
3	from django.db.models import signals
4	from django.dispatch import dispatcher
5	from django.utils.datastructures import SortedDict
6
7	import operator
8
9	# For Python 2.3
10	if not hasattr(__builtins__, 'set'):
11	from sets import Set as set
12
13	LOOKUP_SEPARATOR = '__'
14
15	# Size of each "chunk" for get_iterator calls.
16	# Larger values are slightly faster at the expense of more storage space.
17	GET_ITERATOR_CHUNK_SIZE = 100
18
19	####################
20	# HELPER FUNCTIONS #
21	####################
22
23	# Django currently supports two forms of ordering.
24	# Form 1 (deprecated) example:
25	# order_by=(('pub_date', 'DESC'), ('headline', 'ASC'), (None, 'RANDOM'))
26	# Form 2 (new-style) example:
27	# order_by=('-pub_date', 'headline', '?')
28	# Form 1 is deprecated and will no longer be supported for Django's first
29	# official release. The following code converts from Form 1 to Form 2.
30
31	LEGACY_ORDERING_MAPPING = {'ASC': '_', 'DESC': '-_', 'RANDOM': '?'}
32
33	def handle_legacy_orderlist(order_list):
34	if not order_list or isinstance(order_list[0], basestring):
35	return order_list
36	else:
37	import warnings
38	new_order_list = [LEGACY_ORDERING_MAPPING[j.upper()].replace('_', str(i)) for i, j in order_list]
39	warnings.warn("%r ordering syntax is deprecated. Use %r instead." % (order_list, new_order_list), DeprecationWarning)
40	return new_order_list
41
42	def orderfield2column(f, opts):
43	try:
44	return opts.get_field(f, False).column
45	except FieldDoesNotExist:
46	return f
47
48	def orderlist2sql(order_list, opts, prefix=''):
49	if prefix.endswith('.'):
50	prefix = backend.quote_name(prefix[:-1]) + '.'
51	output = []
52	for f in handle_legacy_orderlist(order_list):
53	if f.startswith('-'):
54	output.append('%s%s DESC' % (prefix, backend.quote_name(orderfield2column(f[1:], opts))))
55	elif f == '?':
56	output.append(backend.get_random_function_sql())
57	else:
58	output.append('%s%s ASC' % (prefix, backend.quote_name(orderfield2column(f, opts))))
59	return ', '.join(output)
60
61	def quote_only_if_word(word):
62	if ' ' in word:
63	return word
64	else:
65	return backend.quote_name(word)
66
67	class QuerySet(object):
68	"Represents a lazy database lookup for a set of objects"
69	def __init__(self, model=None):
70	self.model = model
71	self._filters = Q()
72	self._order_by = None # Ordering, e.g. ('date', '-name'). If None, use model's ordering.
73	self._select_related = False # Whether to fill cache for related objects.
74	self._distinct = False # Whether the query should use SELECT DISTINCT.
75	self._select = {} # Dictionary of attname -> SQL.
76	self._where = [] # List of extra WHERE clauses to use.
77	self._params = [] # List of params to use for extra WHERE clauses.
78	self._tables = [] # List of extra tables to use.
79	self._offset = None # OFFSET clause
80	self._limit = None # LIMIT clause
81	self._result_cache = None
82
83	########################
84	# PYTHON MAGIC METHODS #
85	########################
86
87	def __repr__(self):
88	return repr(self._get_data())
89
90	def __len__(self):
91	return len(self._get_data())
92
93	def __iter__(self):
94	return iter(self._get_data())
95
96	def __getitem__(self, k):
97	"Retrieve an item or slice from the set of results."
98	if self._result_cache is None:
99	if isinstance(k, slice):
100	# Offset:
101	if self._offset is None:
102	offset = k.start
103	elif k.start is None:
104	offset = self._offset
105	else:
106	offset = self._offset + k.start
107	# Now adjust offset to the bounds of any existing limit:
108	if self._limit is not None and k.start is not None:
109	limit = self._limit - k.start
110	else:
111	limit = self._limit
112
113	# Limit:
114	if k.stop is not None and k.start is not None:
115	if limit is None:
116	limit = k.stop - k.start
117	else:
118	limit = min((k.stop - k.start), limit)
119	else:
120	if limit is None:
121	limit = k.stop
122	else:
123	if k.stop is not None:
124	limit = min(k.stop, limit)
125
126	if k.step is None:
127	return self._clone(_offset=offset, _limit=limit)
128	else:
129	return list(self._clone(_offset=offset, _limit=limit))[::k.step]
130	else:
131	return self._clone(_offset=k, _limit=1).get()
132	else:
133	return self._result_cache[k]
134
135	def __and__(self, other):
136	combined = self._combine(other)
137	combined._filters = self._filters & other._filters
138	return combined
139
140	def __or__(self, other):
141	combined = self._combine(other)
142	combined._filters = self._filters \| other._filters
143	return combined
144
145	####################################
146	# METHODS THAT DO DATABASE QUERIES #
147	####################################
148
149	def iterator(self):
150	"Performs the SELECT database lookup of this QuerySet."
151	# self._select is a dictionary, and dictionaries' key order is
152	# undefined, so we convert it to a list of tuples.
153	extra_select = self._select.items()
154
155	cursor = connection.cursor()
156	select, sql, params = self._get_sql_clause()
157	cursor.execute("SELECT " + (self._distinct and "DISTINCT " or "") + ",".join(select) + sql, params)
158	fill_cache = self._select_related
159	index_end = len(self.model._meta.fields)
160	while 1:
161	rows = cursor.fetchmany(GET_ITERATOR_CHUNK_SIZE)
162	if not rows:
163	raise StopIteration
164	for row in rows:
165	if fill_cache:
166	obj, index_end = get_cached_row(self.model, row, 0)
167	else:
168	obj = self.model(*row[:index_end])
169	for i, k in enumerate(extra_select):
170	setattr(obj, k[0], row[index_end+i])
171	yield obj
172
173	def count(self):
174	"Performs a SELECT COUNT() and returns the number of records as an integer."
175	counter = self._clone()
176	counter._order_by = ()
177	counter._offset = None
178	counter._limit = None
179	counter._select_related = False
180	select, sql, params = counter._get_sql_clause()
181	cursor = connection.cursor()
182	cursor.execute("SELECT COUNT(*)" + sql, params)
183	return cursor.fetchone()[0]
184
185	def get(self, args, *kwargs):
186	"Performs the SELECT and returns a single object matching the given keyword arguments."
187	clone = self.filter(args, *kwargs)
188	if not clone._order_by:
189	clone._order_by = ()
190	obj_list = list(clone)
191	if len(obj_list) < 1:
192	raise self.model.DoesNotExist, "%s does not exist for %s" % (self.model._meta.object_name, kwargs)
193	assert len(obj_list) == 1, "get() returned more than one %s -- it returned %s! Lookup parameters were %s" % (self.model._meta.object_name, len(obj_list), kwargs)
194	return obj_list[0]
195
196	def latest(self, field_name=None):
197	"""
198	Returns the latest object, according to the model's 'get_latest_by'
199	option or optional given field_name.
200	"""
201	latest_by = field_name or self.model._meta.get_latest_by
202	assert bool(latest_by), "latest() requires either a field_name parameter or 'get_latest_by' in the model"
203	assert self._limit is None and self._offset is None, \
204	"Cannot change a query once a slice has been taken."
205	return self._clone(_limit=1, _order_by=('-'+latest_by,)).get()
206
207	def in_bulk(self, id_list):
208	"""
209	Returns a dictionary mapping each of the given IDs to the object with
210	that ID.
211	"""
212	assert self._limit is None and self._offset is None, \
213	"Cannot use 'limit' or 'offset' with in_bulk"
214	assert isinstance(id_list, (tuple, list)), "in_bulk() must be provided with a list of IDs."
215	id_list = list(id_list)
216	if id_list == []:
217	return {}
218	qs = self._clone()
219	qs._where.append("%s.%s IN (%s)" % (backend.quote_name(self.model._meta.db_table), backend.quote_name(self.model._meta.pk.column), ",".join(['%s'] * len(id_list))))
220	qs._params.extend(id_list)
221	return dict([(obj._get_pk_val(), obj) for obj in qs.iterator()])
222
223	def delete(self):
224	"""
225	Deletes the records in the current QuerySet.
226	"""
227	assert self._limit is None and self._offset is None, \
228	"Cannot use 'limit' or 'offset' with delete."
229
230	del_query = self._clone()
231
232	# disable non-supported fields
233	del_query._select_related = False
234	del_query._order_by = []
235
236	# Delete objects in chunks to prevent an the list of
237	# related objects from becoming too long
238	more_objects = True
239	while more_objects:
240	# Collect all the objects to be deleted in this chunk, and all the objects
241	# that are related to the objects that are to be deleted
242	seen_objs = SortedDict()
243	more_objects = False
244	for object in del_query[0:GET_ITERATOR_CHUNK_SIZE]:
245	more_objects = True
246	object._collect_sub_objects(seen_objs)
247
248	# If one or more objects were found, delete them.
249	# Otherwise, stop looping.
250	if more_objects:
251	delete_objects(seen_objs)
252
253	# Clear the result cache, in case this QuerySet gets reused.
254	self._result_cache = None
255	delete.alters_data = True
256
257	##################################################
258	# PUBLIC METHODS THAT RETURN A QUERYSET SUBCLASS #
259	##################################################
260
261	def values(self, *fields):
262	return self._clone(klass=ValuesQuerySet, _fields=fields)
263
264	def dates(self, field_name, kind, order='ASC'):
265	"""
266	Returns a list of datetime objects representing all available dates
267	for the given field_name, scoped to 'kind'.
268	"""
269	assert kind in ("month", "year", "day"), "'kind' must be one of 'year', 'month' or 'day'."
270	assert order in ('ASC', 'DESC'), "'order' must be either 'ASC' or 'DESC'."
271	# Let the FieldDoesNotExist exception propagate.
272	field = self.model._meta.get_field(field_name, many_to_many=False)
273	assert isinstance(field, DateField), "%r isn't a DateField." % field_name
274	return self._clone(klass=DateQuerySet, _field=field, _kind=kind, _order=order)
275
276	##################################################################
277	# PUBLIC METHODS THAT ALTER ATTRIBUTES AND RETURN A NEW QUERYSET #
278	##################################################################
279
280	def filter(self, args, *kwargs):
281	"Returns a new QuerySet instance with the args ANDed to the existing set."
282	return self._filter_or_exclude(Q, args, *kwargs)
283
284	def exclude(self, args, *kwargs):
285	"Returns a new QuerySet instance with NOT (args) ANDed to the existing set."
286	return self._filter_or_exclude(QNot, args, *kwargs)
287
288	def _filter_or_exclude(self, qtype, args, *kwargs):
289	if len(args) > 0 or len(kwargs) > 0:
290	assert self._limit is None and self._offset is None, \
291	"Cannot filter a query once a slice has been taken."
292
293	clone = self._clone()
294	if len(kwargs) > 0:
295	clone._filters = clone._filters & qtype(**kwargs)
296	if len(args) > 0:
297	clone._filters = clone._filters & reduce(operator.and_, args)
298	return clone
299
300	def select_related(self, true_or_false=True):
301	"Returns a new QuerySet instance with '_select_related' modified."
302	return self._clone(_select_related=true_or_false)
303
304	def order_by(self, *field_names):
305	"Returns a new QuerySet instance with the ordering changed."
306	assert self._limit is None and self._offset is None, \
307	"Cannot reorder a query once a slice has been taken."
308	return self._clone(_order_by=field_names)
309
310	def distinct(self, true_or_false=True):
311	"Returns a new QuerySet instance with '_distinct' modified."
312	return self._clone(_distinct=true_or_false)
313
314	def extra(self, select=None, where=None, params=None, tables=None):
315	assert self._limit is None and self._offset is None, \
316	"Cannot change a query once a slice has been taken"
317	clone = self._clone()
318	if select: clone._select.update(select)
319	if where: clone._where.extend(where)
320	if params: clone._params.extend(params)
321	if tables: clone._tables.extend(tables)
322	return clone
323
324	###################
325	# PRIVATE METHODS #
326	###################
327
328	def _clone(self, klass=None, **kwargs):
329	if klass is None:
330	klass = self.__class__
331	c = klass()
332	c.model = self.model
333	c._filters = self._filters
334	c._order_by = self._order_by
335	c._select_related = self._select_related
336	c._distinct = self._distinct
337	c._select = self._select.copy()
338	c._where = self._where[:]
339	c._params = self._params[:]
340	c._tables = self._tables[:]
341	c._offset = self._offset
342	c._limit = self._limit
343	c.__dict__.update(kwargs)
344	return c
345
346	def _combine(self, other):
347	assert self._limit is None and self._offset is None \
348	and other._limit is None and other._offset is None, \
349	"Cannot combine queries once a slice has been taken."
350	assert self._distinct == other._distinct, \
351	"Cannot combine a unique query with a non-unique query"
352	# use 'other's order by
353	# (so that A.filter(args1) & A.filter(args2) does the same as
354	# A.filter(args1).filter(args2)
355	combined = other._clone()
356	# If 'self' is ordered and 'other' isn't, propagate 'self's ordering
357	if (self._order_by is not None and len(self._order_by) > 0) and \
358	(combined._order_by is None or len(combined._order_by) == 0):
359	combined._order_by = self._order_by
360	return combined
361
362	def _get_data(self):
363	if self._result_cache is None:
364	self._result_cache = list(self.iterator())
365	return self._result_cache
366
367	def _get_sql_clause(self):
368	opts = self.model._meta
369
370	# Construct the fundamental parts of the query: SELECT X FROM Y WHERE Z.
371	select = ["%s.%s" % (backend.quote_name(opts.db_table), backend.quote_name(f.column)) for f in opts.fields]
372	tables = [quote_only_if_word(t) for t in self._tables]
373	joins = SortedDict()
374	where = self._where[:]
375	params = self._params[:]
376
377	# Convert self._filters into SQL.
378	tables2, joins2, where2, params2 = self._filters.get_sql(opts)
379	tables.extend(tables2)
380	joins.update(joins2)
381	where.extend(where2)
382	params.extend(params2)
383
384	# Add additional tables and WHERE clauses based on select_related.
385	if self._select_related:
386	fill_table_cache(opts, select, tables, where, opts.db_table, [opts.db_table])
387
388	# Add any additional SELECTs.
389	if self._select:
390	select.extend(['(%s) AS %s' % (quote_only_if_word(s[1]), backend.quote_name(s[0])) for s in self._select.items()])
391
392	# Start composing the body of the SQL statement.
393	sql = [" FROM", backend.quote_name(opts.db_table)]
394
395	# Compose the join dictionary into SQL describing the joins.
396	if joins:
397	sql.append(" ".join(["%s %s AS %s ON %s" % (join_type, table, alias, condition)
398	for (alias, (table, join_type, condition)) in joins.items()]))
399
400	# Compose the tables clause into SQL.
401	if tables:
402	sql.append(", " + ", ".join(tables))
403
404	# Compose the where clause into SQL.
405	if where:
406	sql.append(where and "WHERE " + " AND ".join(where))
407
408	# ORDER BY clause
409	order_by = []
410	if self._order_by is not None:
411	ordering_to_use = self._order_by
412	else:
413	ordering_to_use = opts.ordering
414	for f in handle_legacy_orderlist(ordering_to_use):
415	if f == '?': # Special case.
416	order_by.append(backend.get_random_function_sql())
417	else:
418	if f.startswith('-'):
419	col_name = f[1:]
420	order = "DESC"
421	else:
422	col_name = f
423	order = "ASC"
424	if "." in col_name:
425	table_prefix, col_name = col_name.split('.', 1)
426	table_prefix = backend.quote_name(table_prefix) + '.'
427	else:
428	# Use the database table as a column prefix if it wasn't given,
429	# and if the requested column isn't a custom SELECT.
430	if "." not in col_name and col_name not in (self._select or ()):
431	table_prefix = backend.quote_name(opts.db_table) + '.'
432	else:
433	table_prefix = ''
434	order_by.append('%s%s %s' % (table_prefix, backend.quote_name(orderfield2column(col_name, opts)), order))
435	if order_by:
436	sql.append("ORDER BY " + ", ".join(order_by))
437
438	# LIMIT and OFFSET clauses
439	if self._limit is not None:
440	sql.append("%s " % backend.get_limit_offset_sql(self._limit, self._offset))
441	else:
442	assert self._offset is None, "'offset' is not allowed without 'limit'"
443
444	return select, " ".join(sql), params
445
446	class ValuesQuerySet(QuerySet):
447	def iterator(self):
448	# select_related and select aren't supported in values().
449	self._select_related = False
450	self._select = {}
451
452	# self._fields is a list of field names to fetch.
453	if self._fields:
454	columns = [self.model._meta.get_field(f, many_to_many=False).column for f in self._fields]
455	field_names = self._fields
456	else: # Default to all fields.
457	columns = [f.column for f in self.model._meta.fields]
458	field_names = [f.attname for f in self.model._meta.fields]
459
460	cursor = connection.cursor()
461	select, sql, params = self._get_sql_clause()
462	select = ['%s.%s' % (backend.quote_name(self.model._meta.db_table), backend.quote_name(c)) for c in columns]
463	cursor.execute("SELECT " + (self._distinct and "DISTINCT " or "") + ",".join(select) + sql, params)
464	while 1:
465	rows = cursor.fetchmany(GET_ITERATOR_CHUNK_SIZE)
466	if not rows:
467	raise StopIteration
468	for row in rows:

Code

root/django/branches/magic-removal/django/db/models/query.py