Ticket #11863: 11863-rc1.diff
File 11863-rc1.diff, 25.9 KB (added by , 15 years ago) |
---|
-
django/db/models/manager.py
diff --git a/django/db/models/manager.py b/django/db/models/manager.py index 7487fa0..d303682 100644
a b 1 1 import copy 2 from django.db.models.query import QuerySet, EmptyQuerySet, insert_query 2 from django.db.models.query import QuerySet, EmptyQuerySet, insert_query, RawQuerySet 3 3 from django.db.models import signals 4 4 from django.db.models.fields import FieldDoesNotExist 5 5 … … class Manager(object): 181 181 def _update(self, values, **kwargs): 182 182 return self.get_query_set()._update(values, **kwargs) 183 183 184 def raw(self, query, params=None, *args, **kwargs): 185 return RawQuerySet(model=self.model, query=query, params=params, *args, **kwargs) 186 184 187 class ManagerDescriptor(object): 185 188 # This class ensures managers aren't accessible via model instances. 186 189 # For example, Poll.objects works, but poll_obj.objects raises AttributeError. -
django/db/models/query.py
diff --git a/django/db/models/query.py b/django/db/models/query.py index 6a16ce1..2d60fc3 100644
a b class QuerySet(object): 287 287 Returns a dictionary containing the calculations (aggregation) 288 288 over the current queryset 289 289 290 If args is present the expression is passed as a kwarg us sing290 If args is present the expression is passed as a kwarg using 291 291 the Aggregate object's default alias. 292 292 """ 293 293 for arg in args: … … def delete_objects(seen_objs): 1105 1105 if forced_managed: 1106 1106 transaction.leave_transaction_management() 1107 1107 1108 class InsuficientFields(Exception): 1109 """ 1110 The query passed to Queryset.raw doesn't include all of the fields needed 1111 to build a model instance. 1112 """ 1113 pass 1114 1115 class RawQuerySet(object): 1116 """ 1117 Provides an iterator which converts the results of raw SQL queries into 1118 annotated model instances. 1119 """ 1120 def __init__(self, query, model=None, query_obj=None, params=None, translations=None): 1121 self.model = model 1122 self.query = query_obj or sql.RawQuery(sql=query, connection=connection, params=params) 1123 self.params = params or () 1124 self.translations = translations or {} 1125 1126 def __iter__(self): 1127 for row in self.query: 1128 yield self.transform_results(row) 1129 1130 def __len__(self): 1131 return len(self.query) 1132 1133 def __repr__(self): 1134 return "<RawQuerySet: %r>" % (self.query.sql % self.params) 1135 1136 @property 1137 def columns(self): 1138 """ 1139 A list of model field names in the order they'll appear in the 1140 query results. 1141 """ 1142 if not hasattr(self, '_columns'): 1143 self._columns = self.query.get_columns() 1144 1145 # Adjust any column names which don't match field names 1146 for (query_name, model_name) in self.translations.items(): 1147 try: 1148 index = self._columns.index(query_name) 1149 self._columns[index] = model_name 1150 except ValueError: 1151 # Ignore translations for non-existant column names 1152 pass 1153 1154 return self._columns 1155 1156 @property 1157 def model_fields(self): 1158 """ 1159 A dict mapping column names to model field names. 1160 """ 1161 if not hasattr(self, '_model_fields'): 1162 self._model_fields = {} 1163 for field in self.model._meta.fields: 1164 name, column = field.get_attname_column() 1165 self._model_fields[column] = name 1166 return self._model_fields 1167 1168 def transform_results(self, values): 1169 model_init_kwargs = {} 1170 annotations = () 1171 1172 # Associate fields to values 1173 for pos, value in enumerate(values): 1174 column = self.columns[pos] 1175 1176 # Separate properties from annotations 1177 if column in self.model_fields.keys(): 1178 model_init_kwargs[self.model_fields[column]] = value 1179 else: 1180 annotations += (column, value), 1181 1182 if len(model_init_kwargs) < len(self.model_fields): 1183 missing = [column for column, field in self.model_fields.items() 1184 if field not in model_init_kwargs.keys()] 1185 raise InsuficientFields("The query passed doesn't contain all of " 1186 "the needed fields. Missing fields: %s" % ', '.join(missing)) 1187 1188 # Construct model instance and apply annotations 1189 instance = self.model(**model_init_kwargs) 1190 for field, value in annotations: 1191 setattr(instance, field, value) 1192 1193 return instance 1108 1194 1109 1195 def insert_query(model, values, return_id=False, raw_values=False): 1110 1196 """ -
django/db/models/sql/query.py
diff --git a/django/db/models/sql/query.py b/django/db/models/sql/query.py index 7bc45cb..aa301e8 100644
a b from django.core.exceptions import FieldError 23 23 from datastructures import EmptyResultSet, Empty, MultiJoin 24 24 from constants import * 25 25 26 __all__ = ['Query', 'BaseQuery'] 26 __all__ = ['Query', 'BaseQuery', 'RawQuery', 'InvalidQuery'] 27 28 class InvalidQuery(Exception): 29 """ 30 The query passed to raw isn't a safe query to use with raw. 31 """ 32 pass 33 34 class RawQuery(object): 35 """ 36 A single raw SQL query 37 """ 38 39 def __init__(self, sql, connection, params=None): 40 self.validate_sql(sql) 41 self.params = params or () 42 self.sql = sql 43 self.connection = connection 44 self._cache = None 45 self._count = None 46 47 def get_columns(self): 48 self._populate_cache() 49 return [column_meta[0] for column_meta in self.cursor.description] 50 51 def validate_sql(self, sql): 52 if not sql.lower().strip().startswith('select'): 53 raise InvalidQuery('Raw queries are limited to SELECT queries. Use ' 54 'connection.cursor directly for types of queries.') 55 56 def __len__(self): 57 if self._count is None: 58 self._populate_cache() 59 self._count = self.cursor.rowcount 60 # Handle sqlite's quirky rowcount behavior 61 if self._count == -1: 62 self._populate_cache() 63 self._count = len(self._cache) 64 return self._count 65 66 def __iter__(self): 67 self._populate_cache() 68 for value in self._cache: 69 yield value 70 71 def __repr__(self): 72 return "<RawQuery: %r>" % (self.sql % self.params) 73 74 def _populate_cache(self): 75 if self._cache is None: 76 self.cursor = connection.cursor() 77 self.cursor.execute(self.sql, self.params) 78 self._cache = self.cursor.fetchall() 27 79 28 80 class BaseQuery(object): 29 81 """ -
docs/topics/db/queries.txt
diff --git a/docs/topics/db/queries.txt b/docs/topics/db/queries.txt index 968ea7f..1f7e2a1 100644
a b Falling back to raw SQL 1059 1059 ======================= 1060 1060 1061 1061 If you find yourself needing to write an SQL query that is too complex for 1062 Django's database-mapper to handle, you can fall back into raw-SQL statement 1063 mode. 1064 1065 The preferred way to do this is by giving your model custom methods or custom 1066 manager methods that execute queries. Although there's nothing in Django that 1067 *requires* database queries to live in the model layer, this approach keeps all 1068 your data-access logic in one place, which is smart from an code-organization 1069 standpoint. For instructions, see :ref:`topics-db-sql`. 1062 Django's database-mapper to handle, you can fall back on writing SQL by hand. 1063 Django has a couple of options for writing raw SQL queries; see 1064 :ref:`topics-db-sql`. 1070 1065 1071 1066 Finally, it's important to note that the Django database layer is merely an 1072 1067 interface to your database. You can access your database via other tools, -
docs/topics/db/sql.txt
diff --git a/docs/topics/db/sql.txt b/docs/topics/db/sql.txt index 9c53470..b3e8053 100644
a b 1 1 .. _topics-db-sql: 2 2 3 ========================== 3 4 Performing raw SQL queries 4 5 ========================== 5 6 6 Feel free to write custom SQL statements in custom model methods and 7 module-level methods. The object ``django.db.connection`` represents the 7 .. currentmodule:: django.db.models 8 9 When the :ref:`model query APIs <topics-db-queries>` don't go far enough, you 10 can fall back to writing raw SQL. Django gives you two ways of performing raw 11 SQL queries: you can use :meth:`Manager.raw()` to `perform raw queries and 12 return model instances`__, or you can avoid the model layer entirely and 13 `execute custom SQL directly`__. 14 15 __ `performing raw queries`_ 16 __ `executing custom SQL directly`_ 17 18 Performing raw queries 19 ====================== 20 21 .. versionadded:: 1.2 22 23 The ``raw()`` manager method can be used to perform raw SQL queries that 24 return model instances: 25 26 .. method:: Manager.raw(query, params=None, translations=None) 27 28 This method method takes a raw SQL query, executes it, and returns model 29 instances. 30 31 This is best illustrated with an example. Suppose you've got the following model:: 32 33 class Person(models.Model): 34 first_name = models.CharField(...) 35 last_name = models.CharField(...) 36 birth_date = models.DateField(...) 37 38 You could then execute custom SQL like so:: 39 40 >>> Person.objects.raw('SELECT * from myapp_person') 41 [<Person: John Doe>, <Person: Jane Doe>, ...] 42 43 .. admonition:: Model table names 44 45 Where'd the name of the ``Person`` table come from in that example? 46 47 By default, Django figures out a database table name by joining the 48 model's "app label" -- the name you used in ``manage.py startapp`` -- to 49 the model's class name, with an underscore between them. In the example 50 we've assumed that the ``Person`` model lives in an app named ``myapp``, 51 so its table would be ``myapp_person``. 52 53 For more details check out the documentation for the 54 :attr:`~Options.db_table` option, which also lets you manually set the 55 database table name. 56 57 Of course, this example isn't very exciting -- it's exactly the same as 58 running ``Person.objects.all()``. However, ``raw()`` has a bunch of other 59 options that make it very powerful. 60 61 Mapping query fields to model fields 62 ------------------------------------ 63 64 ``raw()`` automatically maps fields in the query to fields on the model. 65 66 The order of fields in your query doesn't matter. In other words, both 67 of the following queries work identically:: 68 69 >>> Person.objects.raw('SELECT first_name, last_name, birth_date FROM myapp_person') 70 ... 71 >>> Person.objects.raw('SELECT last_name, birth_date, first_name FROM myapp_person') 72 ... 73 74 Matching is done by name. This means that you can use SQL's ``AS`` clauses to 75 map fields in the query to model fields. So if you had some other table that 76 had ``Person`` data in it, you could easily map it into ``Person`` instances:: 77 78 >>> Person.objects.raw('''SELECT first AS first_name, 79 ... last AS last_name, 80 ... bd AS birth_date 81 ... FROM some_other_table) 82 83 As long as the names match, the model instances will be created correctly. 84 85 Alternatively, you can map fields in the query to model fields using the 86 ``translations`` argument to ``raw()``. This is a dictionary mapping names of 87 fields in the query to names of fields on the model. For example, the above 88 query could also be written:: 89 90 >>> name_map = {'first': 'first_name', 'last': 'last_name', 'bd': 'birth_date'} 91 >>> Person.objects.raw('SELECT * FROM some_other_table', translations=name_map) 92 93 Fields may also be left out:: 94 95 >>> people = Person.objects.raw('SELECT first_name FROM myapp_person'): 96 97 The ``Person`` objects returned by this query will have their ``first_name`` 98 attributes set correctly, but *will not have any other model fields set*. This 99 means that accessing ``last_name`` or ``birth_date`` will result in an 100 ``AttributeError``. 101 102 You can also execute queries containing fields that aren't defined on the 103 model. For example, we could use `PostgreSQL's age() function`__ to get a list 104 of people with their ages calculated by the database:: 105 106 >>> people = Person.objects.raw('SELECT *, age(birth_date) AS age FROM myapp_person') 107 >>> for p in people: 108 ... print "%s is %s." % (p.first_name, p.age) 109 John is 37. 110 Jane is 42. 111 ... 112 113 __ http://www.postgresql.org/docs/8.4/static/functions-datetime.html 114 115 Passing parameters into ``raw()`` 116 --------------------------------- 117 118 If you need to perform parameterized queries, you can use the ``params`` 119 argument to ``raw()``:: 120 121 >>> lname = 'Doe' 122 >>> Person.objects.raw('SELECT * FROM myapp_person WHERE last_name = %s', [lname]) 123 124 ``params`` is a list of parameters. You'll use ``%s`` placeholders in the 125 query string; they'll be replaced with parameters from the ``params`` list. 126 127 .. warning:: 128 129 **Do not use string formatting on raw queries!** 130 131 It's tempting to write the above query as:: 132 133 >>> query = 'SELECT * FROM myapp_person WHERE last_name = %s', % lname) 134 >>> Person.objects.raw(query) 135 136 **Don't.** 137 138 Using the ``params`` list completely protects you from `SQL injection 139 attacks`__`, a common exploit where attackers inject arbitrary SQL into 140 your database. If you use string interpolation, sooner or later you'll 141 fall victim to SQL injection. As long as you remember to always use the 142 ``params`` list you'll be protected. 143 144 __ http://en.wikipedia.org/wiki/SQL_injection 145 146 Executing custom SQL directly 147 ============================= 148 149 Sometimes even :meth:`Manager.raw` isn't quite enough: you might need to 150 perform queries that don't map cleanly to models, or directly execute 151 ``UPDATE``, ``INSERT``, or ``DELETE`` queries. 152 153 In these cases, you can always access the database directly, routing around 154 the model layer entirely. 155 156 The object ``django.db.connection`` represents the 8 157 current database connection, and ``django.db.transaction`` represents the 9 158 current database transaction. To use the database connection, call 10 159 ``connection.cursor()`` to get a cursor object. Then, call … … changing operation, you should then call 15 164 to the database. If your query is purely a data retrieval operation, no commit 16 165 is required. For example:: 17 166 18 def my_custom_sql( self):167 def my_custom_sql(): 19 168 from django.db import connection, transaction 20 169 cursor = connection.cursor() 21 170 … … necessary. (Also note that Django expects the ``"%s"`` placeholder, *not* the 78 227 ``"?"`` placeholder, which is used by the SQLite Python bindings. This is for 79 228 the sake of consistency and sanity.) 80 229 81 An easier option?82 -----------------83 84 A final note: If all you want to do is a custom ``WHERE`` clause, you can just85 use the ``where``, ``tables`` and ``params`` arguments to the86 :ref:`extra clause <extra>` in the standard queryset API.87 88 230 .. _Python DB-API: http://www.python.org/peps/pep-0249.html 89 231 -
new file tests/modeltests/raw_query/fixtures/initial_data.json
diff --git a/tests/modeltests/raw_query/__init__.py b/tests/modeltests/raw_query/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/modeltests/raw_query/fixtures/initial_data.json b/tests/modeltests/raw_query/fixtures/initial_data.json new file mode 100644 index 0000000..3ff9810
- + 1 [ 2 { 3 "pk": 1, 4 "model": "raw_query.author", 5 "fields": { 6 "dob": "1950-09-20", 7 "first_name": "Joe", 8 "last_name": "Smith" 9 } 10 }, 11 { 12 "pk": 2, 13 "model": "raw_query.author", 14 "fields": { 15 "dob": "1920-04-02", 16 "first_name": "Jill", 17 "last_name": "Doe" 18 } 19 }, 20 { 21 "pk": 3, 22 "model": "raw_query.author", 23 "fields": { 24 "dob": "1986-01-25", 25 "first_name": "Bob", 26 "last_name": "Smith" 27 } 28 }, 29 { 30 "pk": 4, 31 "model": "raw_query.author", 32 "fields": { 33 "dob": "1932-05-10", 34 "first_name": "Bill", 35 "last_name": "Jones" 36 } 37 }, 38 { 39 "pk": 1, 40 "model": "raw_query.book", 41 "fields": { 42 "author": 1, 43 "title": "The awesome book" 44 } 45 }, 46 { 47 "pk": 2, 48 "model": "raw_query.book", 49 "fields": { 50 "author": 1, 51 "title": "The horrible book" 52 } 53 }, 54 { 55 "pk": 3, 56 "model": "raw_query.book", 57 "fields": { 58 "author": 1, 59 "title": "Another awesome book" 60 } 61 }, 62 { 63 "pk": 4, 64 "model": "raw_query.book", 65 "fields": { 66 "author": 3, 67 "title": "Some other book" 68 } 69 }, 70 { 71 "pk": 1, 72 "model": "raw_query.coffee", 73 "fields": { 74 "brand": "dunkin doughnuts" 75 } 76 }, 77 { 78 "pk": 2, 79 "model": "raw_query.coffee", 80 "fields": { 81 "brand": "starbucks" 82 } 83 }, 84 { 85 "pk": 1, 86 "model": "raw_query.reviewer", 87 "fields": { 88 "reviewed": [ 89 2, 90 3, 91 4 92 ] 93 } 94 }, 95 { 96 "pk": 2, 97 "model": "raw_query.reviewer", 98 "fields": { 99 "reviewed": [] 100 } 101 } 102 ] -
new file tests/modeltests/raw_query/models.py
diff --git a/tests/modeltests/raw_query/models.py b/tests/modeltests/raw_query/models.py new file mode 100644 index 0000000..23de97a
- + 1 from django.db import models 2 3 class Author(models.Model): 4 first_name = models.CharField(max_length=255) 5 last_name = models.CharField(max_length=255) 6 dob = models.DateField() 7 8 def __init__(self, *args, **kwargs): 9 super(Author, self).__init__(*args, **kwargs) 10 11 # Perform a check for missing params to protect against real fields 12 # being added as annotations. 13 if len(args) == 0: 14 expected_keywords = ['dob', 'first_name', 'last_name'] 15 for keyword in expected_keywords: 16 if keyword not in kwargs.keys(): 17 raise Exception('All fields are required. %s is missing' % 18 keyword) 19 20 21 22 class Book(models.Model): 23 title = models.CharField(max_length=255) 24 author = models.ForeignKey(Author) 25 26 class Coffee(models.Model): 27 brand = models.CharField(max_length=255, db_column="name") 28 29 class Reviewer(models.Model): 30 reviewed = models.ManyToManyField(Book) 31 No newline at end of file -
new file tests/modeltests/raw_query/tests.py
diff --git a/tests/modeltests/raw_query/tests.py b/tests/modeltests/raw_query/tests.py new file mode 100644 index 0000000..7662444
- + 1 from django.test import TestCase 2 from datetime import datetime 3 from models import Author, Book, Coffee, Reviewer 4 5 from django.db.models.query import InsuficientFields 6 from django.db.models.sql.query import InvalidQuery 7 8 class RawQueryTests(TestCase): 9 10 def assertSuccessfulRawQuery(self, model, query, expected_results, \ 11 expected_annotations=(), params=[], translations=None): 12 """ 13 Execute the passed query against the passed model and check the output 14 """ 15 results = model.objects.raw(query=query, params=params, \ 16 translations=translations) 17 self.assertProcessed(results, expected_results, expected_annotations) 18 self.assertAnnotations(results, expected_annotations) 19 20 def assertProcessed(self, results, orig, expected_annotations=()): 21 """ 22 Compare the results of a raw query against expected results 23 """ 24 self.assertEqual(len(results), len(orig)) 25 for index, item in enumerate(results): 26 orig_item = orig[index] 27 for annotation in expected_annotations: 28 setattr(orig_item, *annotation) 29 30 self.assertEqual(item.id, orig_item.id) 31 32 def assertNoAnnotations(self, results): 33 """ 34 Check that the results of a raw query contain no annotations 35 """ 36 self.assertAnnotations(results, ()) 37 38 def assertAnnotations(self, results, expected_annotations): 39 """ 40 Check that the passed raw query results contain the expected 41 annotations 42 """ 43 if expected_annotations: 44 for index, result in enumerate(results): 45 annotation, value = expected_annotations[index] 46 self.assertTrue(hasattr(result, annotation)) 47 self.assertEqual(getattr(result, annotation), value) 48 49 def testSimpleRawQuery(self): 50 """ 51 Basic test of raw query with a simple database query 52 """ 53 query = "SELECT * FROM raw_query_author" 54 authors = Author.objects.all() 55 self.assertSuccessfulRawQuery(Author, query, authors) 56 57 def testRawQueryLazy(self): 58 """ 59 Raw queries are lazy: they aren't actually executed until they're 60 iterated over. 61 """ 62 q = Author.objects.raw('SELECT * FROM raw_query_author') 63 self.assert_(q.query._cache is None) 64 list(q) 65 self.assert_(q.query._cache is not None) 66 67 def testFkeyRawQuery(self): 68 """ 69 Test of a simple raw query against a model containing a foreign key 70 """ 71 query = "SELECT * FROM raw_query_book" 72 books = Book.objects.all() 73 self.assertSuccessfulRawQuery(Book, query, books) 74 75 def testDBColumnHandler(self): 76 """ 77 Test of a simple raw query against a model containing a field with 78 db_column defined. 79 """ 80 query = "SELECT * FROM raw_query_coffee" 81 coffees = Coffee.objects.all() 82 self.assertSuccessfulRawQuery(Coffee, query, coffees) 83 84 def testOrderHandler(self): 85 """ 86 Test of raw raw query's tolerance for columns being returned in any 87 order 88 """ 89 selects = ( 90 ('dob, last_name, first_name, id'), 91 ('last_name, dob, first_name, id'), 92 ('first_name, last_name, dob, id'), 93 ) 94 95 for select in selects: 96 query = "SELECT %s FROM raw_query_author" % select 97 authors = Author.objects.all() 98 self.assertSuccessfulRawQuery(Author, query, authors) 99 100 def testTranslations(self): 101 """ 102 Test of raw query's optional ability to translate unexpected result 103 column names to specific model fields 104 """ 105 query = "SELECT first_name AS first, last_name AS last, dob, id FROM raw_query_author" 106 translations = {'first': 'first_name', 'last': 'last_name'} 107 authors = Author.objects.all() 108 self.assertSuccessfulRawQuery(Author, query, authors, translations=translations) 109 110 def testParams(self): 111 """ 112 Test passing optional query parameters 113 """ 114 query = "SELECT * FROM raw_query_author WHERE first_name = %s" 115 author = Author.objects.all()[2] 116 params = [author.first_name] 117 results = Author.objects.raw(query=query, params=params) 118 self.assertProcessed(results, [author]) 119 self.assertNoAnnotations(results) 120 self.assertEqual(len(results), 1) 121 122 def testManyToMany(self): 123 """ 124 Test of a simple raw query against a model containing a m2m field 125 """ 126 query = "SELECT * FROM raw_query_reviewer" 127 reviewers = Reviewer.objects.all() 128 self.assertSuccessfulRawQuery(Reviewer, query, reviewers) 129 130 def testExtraConversions(self): 131 """ 132 Test to insure that extra translations are ignored. 133 """ 134 query = "SELECT * FROM raw_query_author" 135 translations = {'something': 'else'} 136 authors = Author.objects.all() 137 self.assertSuccessfulRawQuery(Author, query, authors, translations=translations) 138 139 def testInsufficientColumns(self): 140 query = "SELECT first_name, dob FROM raw_query_author" 141 raised = False 142 try: 143 results = Author.objects.raw(query) 144 results_list = list(results) 145 except InsuficientFields: 146 raised = True 147 148 self.assertTrue(raised) 149 150 def testAnnotations(self): 151 query = "SELECT a.*, count(b.id) as book_count FROM raw_query_author a LEFT JOIN raw_query_book b ON a.id = b.author_id GROUP BY a.id, a.first_name, a.last_name, a.dob ORDER BY a.id" 152 expected_annotations = ( 153 ('book_count', 3), 154 ('book_count', 0), 155 ('book_count', 1), 156 ('book_count', 0), 157 ) 158 authors = Author.objects.all() 159 self.assertSuccessfulRawQuery(Author, query, authors, expected_annotations) 160 161 def testInvalidQuery(self): 162 query = "UPDATE raw_query_author SET first_name='thing' WHERE first_name='Joe'" 163 self.assertRaises(InvalidQuery, Author.objects.raw, query) 164 165 def testWhiteSpaceQuery(self): 166 query = " SELECT * FROM raw_query_author" 167 authors = Author.objects.all() 168 self.assertSuccessfulRawQuery(Author, query, authors) 169 170 def testMultipleIterations(self): 171 query = "SELECT * FROM raw_query_author" 172 normal_authors = Author.objects.all() 173 raw_authors = Author.objects.raw(query) 174 175 # First Iteration 176 first_iterations = 0 177 for index, raw_author in enumerate(raw_authors): 178 self.assertEqual(normal_authors[index], raw_author) 179 first_iterations += 1 180 181 # Second Iteration 182 second_iterations = 0 183 for index, raw_author in enumerate(raw_authors): 184 self.assertEqual(normal_authors[index], raw_author) 185 second_iterations += 1 186 187 self.assertEqual(first_iterations, second_iterations)