Ticket #11863: 11863-rc2.diff
File 11863-rc2.diff, 25.3 KB (added by , 15 years ago) |
---|
-
django/db/models/manager.py
diff --git a/django/db/models/manager.py b/django/db/models/manager.py index 7487fa0..d303682 100644
a b 1 1 import copy 2 from django.db.models.query import QuerySet, EmptyQuerySet, insert_query 2 from django.db.models.query import QuerySet, EmptyQuerySet, insert_query, RawQuerySet 3 3 from django.db.models import signals 4 4 from django.db.models.fields import FieldDoesNotExist 5 5 … … class Manager(object): 181 181 def _update(self, values, **kwargs): 182 182 return self.get_query_set()._update(values, **kwargs) 183 183 184 def raw(self, query, params=None, *args, **kwargs): 185 return RawQuerySet(model=self.model, query=query, params=params, *args, **kwargs) 186 184 187 class ManagerDescriptor(object): 185 188 # This class ensures managers aren't accessible via model instances. 186 189 # For example, Poll.objects works, but poll_obj.objects raises AttributeError. -
django/db/models/query.py
diff --git a/django/db/models/query.py b/django/db/models/query.py index 6a16ce1..e2310ed 100644
a b class QuerySet(object): 287 287 Returns a dictionary containing the calculations (aggregation) 288 288 over the current queryset 289 289 290 If args is present the expression is passed as a kwarg us sing290 If args is present the expression is passed as a kwarg using 291 291 the Aggregate object's default alias. 292 292 """ 293 293 for arg in args: … … def delete_objects(seen_objs): 1105 1105 if forced_managed: 1106 1106 transaction.leave_transaction_management() 1107 1107 1108 class RawQuerySet(object): 1109 """ 1110 Provides an iterator which converts the results of raw SQL queries into 1111 annotated model instances. 1112 """ 1113 def __init__(self, query, model=None, query_obj=None, params=None, translations=None): 1114 self.model = model 1115 self.query = query_obj or sql.RawQuery(sql=query, connection=connection, params=params) 1116 self.params = params or () 1117 self.translations = translations or {} 1118 1119 def __iter__(self): 1120 for row in self.query: 1121 yield self.transform_results(row) 1122 1123 def __repr__(self): 1124 return "<RawQuerySet: %r>" % (self.query.sql % self.params) 1125 1126 @property 1127 def columns(self): 1128 """ 1129 A list of model field names in the order they'll appear in the 1130 query results. 1131 """ 1132 if not hasattr(self, '_columns'): 1133 self._columns = self.query.get_columns() 1134 1135 # Adjust any column names which don't match field names 1136 for (query_name, model_name) in self.translations.items(): 1137 try: 1138 index = self._columns.index(query_name) 1139 self._columns[index] = model_name 1140 except ValueError: 1141 # Ignore translations for non-existant column names 1142 pass 1143 1144 return self._columns 1145 1146 @property 1147 def model_fields(self): 1148 """ 1149 A dict mapping column names to model field names. 1150 """ 1151 if not hasattr(self, '_model_fields'): 1152 self._model_fields = {} 1153 for field in self.model._meta.fields: 1154 name, column = field.get_attname_column() 1155 self._model_fields[column] = name 1156 return self._model_fields 1157 1158 def transform_results(self, values): 1159 model_init_kwargs = {} 1160 annotations = () 1161 1162 # Associate fields to values 1163 for pos, value in enumerate(values): 1164 column = self.columns[pos] 1165 1166 # Separate properties from annotations 1167 if column in self.model_fields.keys(): 1168 model_init_kwargs[self.model_fields[column]] = value 1169 else: 1170 annotations += (column, value), 1171 1172 # Construct model instance and apply annotations 1173 instance = self.model(**model_init_kwargs) 1174 for field, value in annotations: 1175 setattr(instance, field, value) 1176 1177 return instance 1108 1178 1109 1179 def insert_query(model, values, return_id=False, raw_values=False): 1110 1180 """ -
django/db/models/sql/query.py
diff --git a/django/db/models/sql/query.py b/django/db/models/sql/query.py index 7bc45cb..6140a8a 100644
a b from django.core.exceptions import FieldError 23 23 from datastructures import EmptyResultSet, Empty, MultiJoin 24 24 from constants import * 25 25 26 __all__ = ['Query', 'BaseQuery'] 26 __all__ = ['Query', 'BaseQuery', 'RawQuery', 'InvalidQuery'] 27 28 class InvalidQuery(Exception): 29 """ 30 The query passed to raw isn't a safe query to use with raw. 31 """ 32 pass 33 34 class RawQuery(object): 35 """ 36 A single raw SQL query 37 """ 38 39 def __init__(self, sql, connection, params=None): 40 self.validate_sql(sql) 41 self.params = params or () 42 self.sql = sql 43 self.connection = connection 44 self.cursor = None 45 46 def get_columns(self): 47 if self.cursor is None: 48 self._execute_query() 49 return [column_meta[0] for column_meta in self.cursor.description] 50 51 def validate_sql(self, sql): 52 if not sql.lower().strip().startswith('select'): 53 raise InvalidQuery('Raw queries are limited to SELECT queries. Use ' 54 'connection.cursor directly for types of queries.') 55 56 def __iter__(self): 57 # Always execute a new query for a new iterator. 58 # This could be optomized with a cache at the expense of RAM. 59 self._execute_query() 60 return self.cursor 61 62 def __repr__(self): 63 return "<RawQuery: %r>" % (self.sql % self.params) 64 65 def _execute_query(self): 66 self.cursor = self.connection.cursor() 67 self.cursor.execute(self.sql, self.params) 27 68 28 69 class BaseQuery(object): 29 70 """ -
docs/topics/db/queries.txt
diff --git a/docs/topics/db/queries.txt b/docs/topics/db/queries.txt index 968ea7f..1f7e2a1 100644
a b Falling back to raw SQL 1059 1059 ======================= 1060 1060 1061 1061 If you find yourself needing to write an SQL query that is too complex for 1062 Django's database-mapper to handle, you can fall back into raw-SQL statement 1063 mode. 1064 1065 The preferred way to do this is by giving your model custom methods or custom 1066 manager methods that execute queries. Although there's nothing in Django that 1067 *requires* database queries to live in the model layer, this approach keeps all 1068 your data-access logic in one place, which is smart from an code-organization 1069 standpoint. For instructions, see :ref:`topics-db-sql`. 1062 Django's database-mapper to handle, you can fall back on writing SQL by hand. 1063 Django has a couple of options for writing raw SQL queries; see 1064 :ref:`topics-db-sql`. 1070 1065 1071 1066 Finally, it's important to note that the Django database layer is merely an 1072 1067 interface to your database. You can access your database via other tools, -
docs/topics/db/sql.txt
diff --git a/docs/topics/db/sql.txt b/docs/topics/db/sql.txt index 9c53470..64a4973 100644
a b 1 1 .. _topics-db-sql: 2 2 3 ========================== 3 4 Performing raw SQL queries 4 5 ========================== 5 6 6 Feel free to write custom SQL statements in custom model methods and 7 module-level methods. The object ``django.db.connection`` represents the 7 .. currentmodule:: django.db.models 8 9 When the :ref:`model query APIs <topics-db-queries>` don't go far enough, you 10 can fall back to writing raw SQL. Django gives you two ways of performing raw 11 SQL queries: you can use :meth:`Manager.raw()` to `perform raw queries and 12 return model instances`__, or you can avoid the model layer entirely and 13 `execute custom SQL directly`__. 14 15 __ `performing raw queries`_ 16 __ `executing custom SQL directly`_ 17 18 Performing raw queries 19 ====================== 20 21 .. versionadded:: 1.2 22 23 The ``raw()`` manager method can be used to perform raw SQL queries that 24 return model instances: 25 26 .. method:: Manager.raw(query, params=None, translations=None) 27 28 This method method takes a raw SQL query, executes it, and returns model 29 instances. 30 31 This is best illustrated with an example. Suppose you've got the following model:: 32 33 class Person(models.Model): 34 first_name = models.CharField(...) 35 last_name = models.CharField(...) 36 birth_date = models.DateField(...) 37 38 You could then execute custom SQL like so:: 39 40 >>> Person.objects.raw('SELECT * from myapp_person') 41 [<Person: John Doe>, <Person: Jane Doe>, ...] 42 43 .. admonition:: Model table names 44 45 Where'd the name of the ``Person`` table come from in that example? 46 47 By default, Django figures out a database table name by joining the 48 model's "app label" -- the name you used in ``manage.py startapp`` -- to 49 the model's class name, with an underscore between them. In the example 50 we've assumed that the ``Person`` model lives in an app named ``myapp``, 51 so its table would be ``myapp_person``. 52 53 For more details check out the documentation for the 54 :attr:`~Options.db_table` option, which also lets you manually set the 55 database table name. 56 57 Of course, this example isn't very exciting -- it's exactly the same as 58 running ``Person.objects.all()``. However, ``raw()`` has a bunch of other 59 options that make it very powerful. 60 61 Mapping query fields to model fields 62 ------------------------------------ 63 64 ``raw()`` automatically maps fields in the query to fields on the model. 65 66 The order of fields in your query doesn't matter. In other words, both 67 of the following queries work identically:: 68 69 >>> Person.objects.raw('SELECT first_name, last_name, birth_date FROM myapp_person') 70 ... 71 >>> Person.objects.raw('SELECT last_name, birth_date, first_name FROM myapp_person') 72 ... 73 74 Matching is done by name. This means that you can use SQL's ``AS`` clauses to 75 map fields in the query to model fields. So if you had some other table that 76 had ``Person`` data in it, you could easily map it into ``Person`` instances:: 77 78 >>> Person.objects.raw('''SELECT first AS first_name, 79 ... last AS last_name, 80 ... bd AS birth_date 81 ... FROM some_other_table) 82 83 As long as the names match, the model instances will be created correctly. 84 85 Alternatively, you can map fields in the query to model fields using the 86 ``translations`` argument to ``raw()``. This is a dictionary mapping names of 87 fields in the query to names of fields on the model. For example, the above 88 query could also be written:: 89 90 >>> name_map = {'first': 'first_name', 'last': 'last_name', 'bd': 'birth_date'} 91 >>> Person.objects.raw('SELECT * FROM some_other_table', translations=name_map) 92 93 Fields may also be left out:: 94 95 >>> people = Person.objects.raw('SELECT first_name FROM myapp_person'): 96 97 The ``Person`` objects returned by this query will have their ``first_name`` 98 attributes set correctly, but all other fields will be set to their default 99 (usually ``None`` or the empty string). For example:: 100 101 >>> for p in Person.objects.raw('SELECT first_name FROM myapp_person'): 102 ... print (p.first_name, p.last_name, p.birth_date) 103 ... 104 ('John', '', None) 105 ('Jane', '', None) 106 107 As you can see, the ``first_name`` attribute is set, but ``last_name`` and 108 ``birth_date`` are blank. You'll also note that ``last_name``, as a 109 :class:`CharField`, has a default value of ``''`` (an empty string); 110 ``birth_date``, on the other hand, gets a default value of ``None`` since it's 111 a :class:`DateField``.` 112 113 You can also execute queries containing fields that aren't defined on the 114 model. For example, we could use `PostgreSQL's age() function`__ to get a list 115 of people with their ages calculated by the database:: 116 117 >>> people = Person.objects.raw('SELECT *, age(birth_date) AS age FROM myapp_person') 118 >>> for p in people: 119 ... print "%s is %s." % (p.first_name, p.age) 120 John is 37. 121 Jane is 42. 122 ... 123 124 __ http://www.postgresql.org/docs/8.4/static/functions-datetime.html 125 126 Passing parameters into ``raw()`` 127 --------------------------------- 128 129 If you need to perform parameterized queries, you can use the ``params`` 130 argument to ``raw()``:: 131 132 >>> lname = 'Doe' 133 >>> Person.objects.raw('SELECT * FROM myapp_person WHERE last_name = %s', [lname]) 134 135 ``params`` is a list of parameters. You'll use ``%s`` placeholders in the 136 query string (regardless of your database engine); they'll be replaced with 137 parameters from the ``params`` list. 138 139 .. warning:: 140 141 **Do not use string formatting on raw queries!** 142 143 It's tempting to write the above query as:: 144 145 >>> query = 'SELECT * FROM myapp_person WHERE last_name = %s', % lname 146 >>> Person.objects.raw(query) 147 148 **Don't.** 149 150 Using the ``params`` list completely protects you from `SQL injection 151 attacks`__`, a common exploit where attackers inject arbitrary SQL into 152 your database. If you use string interpolation, sooner or later you'll 153 fall victim to SQL injection. As long as you remember to always use the 154 ``params`` list you'll be protected. 155 156 __ http://en.wikipedia.org/wiki/SQL_injection 157 158 Executing custom SQL directly 159 ============================= 160 161 Sometimes even :meth:`Manager.raw` isn't quite enough: you might need to 162 perform queries that don't map cleanly to models, or directly execute 163 ``UPDATE``, ``INSERT``, or ``DELETE`` queries. 164 165 In these cases, you can always access the database directly, routing around 166 the model layer entirely. 167 168 The object ``django.db.connection`` represents the 8 169 current database connection, and ``django.db.transaction`` represents the 9 170 current database transaction. To use the database connection, call 10 171 ``connection.cursor()`` to get a cursor object. Then, call … … changing operation, you should then call 15 176 to the database. If your query is purely a data retrieval operation, no commit 16 177 is required. For example:: 17 178 18 def my_custom_sql( self):179 def my_custom_sql(): 19 180 from django.db import connection, transaction 20 181 cursor = connection.cursor() 21 182 … … necessary. (Also note that Django expects the ``"%s"`` placeholder, *not* the 78 239 ``"?"`` placeholder, which is used by the SQLite Python bindings. This is for 79 240 the sake of consistency and sanity.) 80 241 81 An easier option?82 -----------------83 84 A final note: If all you want to do is a custom ``WHERE`` clause, you can just85 use the ``where``, ``tables`` and ``params`` arguments to the86 :ref:`extra clause <extra>` in the standard queryset API.87 88 242 .. _Python DB-API: http://www.python.org/peps/pep-0249.html 89 243 -
new file tests/modeltests/raw_query/fixtures/initial_data.json
diff --git a/tests/modeltests/raw_query/__init__.py b/tests/modeltests/raw_query/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/modeltests/raw_query/fixtures/initial_data.json b/tests/modeltests/raw_query/fixtures/initial_data.json new file mode 100644 index 0000000..3ff9810
- + 1 [ 2 { 3 "pk": 1, 4 "model": "raw_query.author", 5 "fields": { 6 "dob": "1950-09-20", 7 "first_name": "Joe", 8 "last_name": "Smith" 9 } 10 }, 11 { 12 "pk": 2, 13 "model": "raw_query.author", 14 "fields": { 15 "dob": "1920-04-02", 16 "first_name": "Jill", 17 "last_name": "Doe" 18 } 19 }, 20 { 21 "pk": 3, 22 "model": "raw_query.author", 23 "fields": { 24 "dob": "1986-01-25", 25 "first_name": "Bob", 26 "last_name": "Smith" 27 } 28 }, 29 { 30 "pk": 4, 31 "model": "raw_query.author", 32 "fields": { 33 "dob": "1932-05-10", 34 "first_name": "Bill", 35 "last_name": "Jones" 36 } 37 }, 38 { 39 "pk": 1, 40 "model": "raw_query.book", 41 "fields": { 42 "author": 1, 43 "title": "The awesome book" 44 } 45 }, 46 { 47 "pk": 2, 48 "model": "raw_query.book", 49 "fields": { 50 "author": 1, 51 "title": "The horrible book" 52 } 53 }, 54 { 55 "pk": 3, 56 "model": "raw_query.book", 57 "fields": { 58 "author": 1, 59 "title": "Another awesome book" 60 } 61 }, 62 { 63 "pk": 4, 64 "model": "raw_query.book", 65 "fields": { 66 "author": 3, 67 "title": "Some other book" 68 } 69 }, 70 { 71 "pk": 1, 72 "model": "raw_query.coffee", 73 "fields": { 74 "brand": "dunkin doughnuts" 75 } 76 }, 77 { 78 "pk": 2, 79 "model": "raw_query.coffee", 80 "fields": { 81 "brand": "starbucks" 82 } 83 }, 84 { 85 "pk": 1, 86 "model": "raw_query.reviewer", 87 "fields": { 88 "reviewed": [ 89 2, 90 3, 91 4 92 ] 93 } 94 }, 95 { 96 "pk": 2, 97 "model": "raw_query.reviewer", 98 "fields": { 99 "reviewed": [] 100 } 101 } 102 ] -
new file tests/modeltests/raw_query/models.py
diff --git a/tests/modeltests/raw_query/models.py b/tests/modeltests/raw_query/models.py new file mode 100644 index 0000000..fb5503d
- + 1 from django.db import models 2 3 class Author(models.Model): 4 first_name = models.CharField(max_length=255) 5 last_name = models.CharField(max_length=255) 6 dob = models.DateField() 7 8 def __init__(self, *args, **kwargs): 9 super(Author, self).__init__(*args, **kwargs) 10 # Protect against annotations being passed to __init__ -- 11 # this'll make the test suite get angry if annotations aren't 12 # treated differently than fields. 13 for k in kwargs: 14 assert k in [f.attname for f in self._meta.fields], \ 15 "Author.__init__ got an unexpected paramater: %s" % k 16 17 class Book(models.Model): 18 title = models.CharField(max_length=255) 19 author = models.ForeignKey(Author) 20 21 class Coffee(models.Model): 22 brand = models.CharField(max_length=255, db_column="name") 23 24 class Reviewer(models.Model): 25 reviewed = models.ManyToManyField(Book) 26 No newline at end of file -
new file tests/modeltests/raw_query/tests.py
diff --git a/tests/modeltests/raw_query/tests.py b/tests/modeltests/raw_query/tests.py new file mode 100644 index 0000000..fc2d7a9
- + 1 from django.test import TestCase 2 from datetime import datetime 3 from models import Author, Book, Coffee, Reviewer 4 from django.db.models.sql.query import InvalidQuery 5 6 class RawQueryTests(TestCase): 7 8 def assertSuccessfulRawQuery(self, model, query, expected_results, 9 expected_annotations=(), params=[], translations=None): 10 """ 11 Execute the passed query against the passed model and check the output 12 """ 13 results = list(model.objects.raw(query=query, params=params, translations=translations)) 14 self.assertProcessed(results, expected_results, expected_annotations) 15 self.assertAnnotations(results, expected_annotations) 16 17 def assertProcessed(self, results, orig, expected_annotations=()): 18 """ 19 Compare the results of a raw query against expected results 20 """ 21 self.assertEqual(len(results), len(orig)) 22 for index, item in enumerate(results): 23 orig_item = orig[index] 24 for annotation in expected_annotations: 25 setattr(orig_item, *annotation) 26 27 self.assertEqual(item.id, orig_item.id) 28 29 def assertNoAnnotations(self, results): 30 """ 31 Check that the results of a raw query contain no annotations 32 """ 33 self.assertAnnotations(results, ()) 34 35 def assertAnnotations(self, results, expected_annotations): 36 """ 37 Check that the passed raw query results contain the expected 38 annotations 39 """ 40 if expected_annotations: 41 for index, result in enumerate(results): 42 annotation, value = expected_annotations[index] 43 self.assertTrue(hasattr(result, annotation)) 44 self.assertEqual(getattr(result, annotation), value) 45 46 def testSimpleRawQuery(self): 47 """ 48 Basic test of raw query with a simple database query 49 """ 50 query = "SELECT * FROM raw_query_author" 51 authors = Author.objects.all() 52 self.assertSuccessfulRawQuery(Author, query, authors) 53 54 def testRawQueryLazy(self): 55 """ 56 Raw queries are lazy: they aren't actually executed until they're 57 iterated over. 58 """ 59 q = Author.objects.raw('SELECT * FROM raw_query_author') 60 self.assert_(q.query.cursor is None) 61 list(q) 62 self.assert_(q.query.cursor is not None) 63 64 def testFkeyRawQuery(self): 65 """ 66 Test of a simple raw query against a model containing a foreign key 67 """ 68 query = "SELECT * FROM raw_query_book" 69 books = Book.objects.all() 70 self.assertSuccessfulRawQuery(Book, query, books) 71 72 def testDBColumnHandler(self): 73 """ 74 Test of a simple raw query against a model containing a field with 75 db_column defined. 76 """ 77 query = "SELECT * FROM raw_query_coffee" 78 coffees = Coffee.objects.all() 79 self.assertSuccessfulRawQuery(Coffee, query, coffees) 80 81 def testOrderHandler(self): 82 """ 83 Test of raw raw query's tolerance for columns being returned in any 84 order 85 """ 86 selects = ( 87 ('dob, last_name, first_name, id'), 88 ('last_name, dob, first_name, id'), 89 ('first_name, last_name, dob, id'), 90 ) 91 92 for select in selects: 93 query = "SELECT %s FROM raw_query_author" % select 94 authors = Author.objects.all() 95 self.assertSuccessfulRawQuery(Author, query, authors) 96 97 def testTranslations(self): 98 """ 99 Test of raw query's optional ability to translate unexpected result 100 column names to specific model fields 101 """ 102 query = "SELECT first_name AS first, last_name AS last, dob, id FROM raw_query_author" 103 translations = {'first': 'first_name', 'last': 'last_name'} 104 authors = Author.objects.all() 105 self.assertSuccessfulRawQuery(Author, query, authors, translations=translations) 106 107 def testParams(self): 108 """ 109 Test passing optional query parameters 110 """ 111 query = "SELECT * FROM raw_query_author WHERE first_name = %s" 112 author = Author.objects.all()[2] 113 params = [author.first_name] 114 results = list(Author.objects.raw(query=query, params=params)) 115 self.assertProcessed(results, [author]) 116 self.assertNoAnnotations(results) 117 self.assertEqual(len(results), 1) 118 119 def testManyToMany(self): 120 """ 121 Test of a simple raw query against a model containing a m2m field 122 """ 123 query = "SELECT * FROM raw_query_reviewer" 124 reviewers = Reviewer.objects.all() 125 self.assertSuccessfulRawQuery(Reviewer, query, reviewers) 126 127 def testExtraConversions(self): 128 """ 129 Test to insure that extra translations are ignored. 130 """ 131 query = "SELECT * FROM raw_query_author" 132 translations = {'something': 'else'} 133 authors = Author.objects.all() 134 self.assertSuccessfulRawQuery(Author, query, authors, translations=translations) 135 136 def testMissingFields(self): 137 query = "SELECT first_name, dob FROM raw_query_author" 138 for author in Author.objects.raw(query): 139 self.assertNotEqual(author.first_name, None) 140 # last_name isn't given, so it's set to the default of '' 141 self.assertEqual(author.last_name, '') 142 143 def testAnnotations(self): 144 query = "SELECT a.*, count(b.id) as book_count FROM raw_query_author a LEFT JOIN raw_query_book b ON a.id = b.author_id GROUP BY a.id, a.first_name, a.last_name, a.dob ORDER BY a.id" 145 expected_annotations = ( 146 ('book_count', 3), 147 ('book_count', 0), 148 ('book_count', 1), 149 ('book_count', 0), 150 ) 151 authors = Author.objects.all() 152 self.assertSuccessfulRawQuery(Author, query, authors, expected_annotations) 153 154 def testInvalidQuery(self): 155 query = "UPDATE raw_query_author SET first_name='thing' WHERE first_name='Joe'" 156 self.assertRaises(InvalidQuery, Author.objects.raw, query) 157 158 def testWhiteSpaceQuery(self): 159 query = " SELECT * FROM raw_query_author" 160 authors = Author.objects.all() 161 self.assertSuccessfulRawQuery(Author, query, authors) 162 163 def testMultipleIterations(self): 164 query = "SELECT * FROM raw_query_author" 165 normal_authors = Author.objects.all() 166 raw_authors = Author.objects.raw(query) 167 168 # First Iteration 169 first_iterations = 0 170 for index, raw_author in enumerate(raw_authors): 171 self.assertEqual(normal_authors[index], raw_author) 172 first_iterations += 1 173 174 # Second Iteration 175 second_iterations = 0 176 for index, raw_author in enumerate(raw_authors): 177 self.assertEqual(normal_authors[index], raw_author) 178 second_iterations += 1 179 180 self.assertEqual(first_iterations, second_iterations) 181 No newline at end of file