Ticket #7596: t7596-alex.diff

File t7596-alex.diff, 24.5 KB (added by Karen Tracey, 13 years ago)

Same patch, extension that trac will hopefully recognize properly

  • django/contrib/auth/management/__init__.py 

    diff --git a/django/contrib/auth/management/__init__.py b/django/contrib/auth/management/__init__.py
index 9966849..876d84d 100644
    a b def create_permissions(app, created_models, verbosity, **kwargs):  
    4646        "content_type", "codename"
    4747    ))
    4848
    49     for ctype, (codename, name) in searched_perms:
    50         # If the permissions exists, move on.
    51         if (ctype.pk, codename) in all_perms:
    52             continue
    53         p = auth_app.Permission.objects.create(
    54             codename=codename,
    55             name=name,
    56             content_type=ctype
    57         )
    58         if verbosity >= 2:
     49    objs = [
     50        auth_app.Permission(codename=codename, name=name, content_type=ctype)
     51        for ctype, (codename, name) in searched_perms
     52        if (ctype.pk, codename) not in all_perms
     53    ]
     54    auth_app.Permission.objects.bulk_create(objs)
     55    if verbosity >= 2:
     56        for obj in objs:
    5957            print "Adding permission '%s'" % p
    6058
    6159

  • django/db/backends/__init__.py 

    diff --git a/django/db/backends/__init__.py b/django/db/backends/__init__.py
index b64fb01..8ed6d6b 100644
    a b class BaseDatabaseFeatures(object):  
    272272
    273273    can_use_chunked_reads = True
    274274    can_return_id_from_insert = False
     275    has_bulk_insert = False
    275276    uses_autocommit = False
    276277    uses_savepoints = False
     278    can_combine_inserts_with_and_without_auto_increment_pk = False
    277279
    278280    # If True, don't use integer foreign keys referring to, e.g., positive
    279281    # integer primary keys.

  • django/db/backends/postgresql_psycopg2/base.py 

    diff --git a/django/db/backends/postgresql_psycopg2/base.py b/django/db/backends/postgresql_psycopg2/base.py
index 67e2877..ad355b8 100644
    a b class DatabaseFeatures(BaseDatabaseFeatures):  
    7272    can_defer_constraint_checks = True
    7373    has_select_for_update = True
    7474    has_select_for_update_nowait = True
     75    has_bulk_insert = True
    7576
    7677
    7778class DatabaseWrapper(BaseDatabaseWrapper):

  • django/db/backends/postgresql_psycopg2/operations.py 

    diff --git a/django/db/backends/postgresql_psycopg2/operations.py b/django/db/backends/postgresql_psycopg2/operations.py
index 3315913..7a9c406 100644
    a b class DatabaseOperations(BaseDatabaseOperations):  
    208208
    209209    def return_insert_id(self):
    210210        return "RETURNING %s", ()
     211
     212    def bulk_insert_sql(self, fields, num_values):
     213        items_sql = "(%s)" % ", ".join(["%s"] * len(fields))
     214        return "VALUES " + ", ".join([items_sql] * num_values)
     215 No newline at end of file

  • django/db/backends/sqlite3/base.py 

    diff --git a/django/db/backends/sqlite3/base.py b/django/db/backends/sqlite3/base.py
index 5b4a1c2..ea65865 100644
    a b class DatabaseFeatures(BaseDatabaseFeatures):  
    5757    supports_unspecified_pk = True
    5858    supports_1000_query_parameters = False
    5959    supports_mixed_date_datetime_comparisons = False
     60    has_bulk_insert = True
     61    can_combine_inserts_with_and_without_auto_increment_pk = True
    6062
    6163    def _supports_stddev(self):
    6264        """Confirm support for STDDEV and related stats functions
    … … class DatabaseOperations(BaseDatabaseOperations):  
    105107        return ""
    106108
    107109    def pk_default_value(self):
    108         return 'NULL'
     110        return "NULL"
    109111
    110112    def quote_name(self, name):
    111113        if name.startswith('"') and name.endswith('"'):
    … … class DatabaseOperations(BaseDatabaseOperations):  
    153155        # No field, or the field isn't known to be a decimal or integer
    154156        return value
    155157
     158    def bulk_insert_sql(self, fields, num_values):
     159        res = []
     160        res.append("SELECT %s" % ", ".join(
     161            "%%s AS %s" % self.quote_name(f.column) for f in fields
     162        ))
     163        res.extend(["UNION SELECT %s" % ", ".join(["%s"] * len(fields))] * (num_values - 1))
     164        return " ".join(res)
     165
    156166class DatabaseWrapper(BaseDatabaseWrapper):
    157167    vendor = 'sqlite'
    158168    # SQLite requires LIKE statements to include an ESCAPE clause if the value

  • django/db/models/base.py 

    diff --git a/django/db/models/base.py b/django/db/models/base.py
index 31310ea..f71f13e 100644
    a b class Model(object):  
    539539                    order_value = manager.using(using).filter(**{field.name: getattr(self, field.attname)}).count()
    540540                    self._order = order_value
    541541
     542                fields = meta.local_fields
    542543                if not pk_set:
    543544                    if force_update:
    544545                        raise ValueError("Cannot force an update in save() with no primary key.")
    545                     values = [(f, f.get_db_prep_save(raw and getattr(self, f.attname) or f.pre_save(self, True), connection=connection))
    546                         for f in meta.local_fields if not isinstance(f, AutoField)]
    547                 else:
    548                     values = [(f, f.get_db_prep_save(raw and getattr(self, f.attname) or f.pre_save(self, True), connection=connection))
    549                         for f in meta.local_fields]
     546                    fields = [f for f in fields if not isinstance(f, AutoField)]
    550547
    551548                record_exists = False
    552549
    553550                update_pk = bool(meta.has_auto_field and not pk_set)
    554                 if values:
    555                     # Create a new record.
    556                     result = manager._insert(values, return_id=update_pk, using=using)
    557                 else:
    558                     # Create a new record with defaults for everything.
    559                     result = manager._insert([(meta.pk, connection.ops.pk_default_value())], return_id=update_pk, raw_values=True, using=using)
     551                result = manager._insert([self], fields=fields, return_id=update_pk, using=using, raw=raw)
    560552
    561553                if update_pk:
    562554                    setattr(self, meta.pk.attname, result)

  • django/db/models/fields/related.py 

    diff --git a/django/db/models/fields/related.py b/django/db/models/fields/related.py
index cedf308..4d6a8a7 100644
    a b class ForeignRelatedObjectsDescriptor(object):  
    432432            add.alters_data = True
    433433
    434434            def create(self, **kwargs):
    435                 kwargs.update({rel_field.name: instance})
     435                kwargs[rel_field.name] = instance
    436436                db = router.db_for_write(rel_model, instance=instance)
    437437                return super(RelatedManager, self.db_manager(db)).create(**kwargs)
    438438            create.alters_data = True
    … … class ForeignRelatedObjectsDescriptor(object):  
    440440            def get_or_create(self, **kwargs):
    441441                # Update kwargs with the related object that this
    442442                # ForeignRelatedObjectsDescriptor knows about.
    443                 kwargs.update({rel_field.name: instance})
     443                kwargs[rel_field.name] = instance
    444444                db = router.db_for_write(rel_model, instance=instance)
    445445                return super(RelatedManager, self.db_manager(db)).get_or_create(**kwargs)
    446446            get_or_create.alters_data = True
    … … def create_many_related_manager(superclass, rel=False):  
    580580                        instance=self.instance, reverse=self.reverse,
    581581                        model=self.model, pk_set=new_ids, using=db)
    582582                # Add the ones that aren't there already
    583                 for obj_id in new_ids:
    584                     self.through._default_manager.using(db).create(**{
     583                self.through._default_manager.using(db).bulk_create([
     584                    self.through(**{
    585585                        '%s_id' % source_field_name: self._pk_val,
    586586                        '%s_id' % target_field_name: obj_id,
    587587                    })
     588                    for obj_id in new_ids
     589                ])
    588590                if self.reverse or source_field_name == self.source_field_name:
    589591                    # Don't send the signal when we are inserting the
    590592                    # duplicate data row for symmetrical reverse entries.
    … … class ReverseManyRelatedObjectsDescriptor(object):  
    703705    def __init__(self, m2m_field):
    704706        self.field = m2m_field
    705707
    706     def _through(self):
     708    @property
     709    def through(self):
    707710        # through is provided so that you have easy access to the through
    708711        # model (Book.authors.through) for inlines, etc. This is done as
    709712        # a property to ensure that the fully resolved value is returned.
    710713        return self.field.rel.through
    711     through = property(_through)
    712714
    713715    def __get__(self, instance, instance_type=None):
    714716        if instance is None:

  • django/db/models/manager.py 

    diff --git a/django/db/models/manager.py b/django/db/models/manager.py
index 4fa4c4a..92090e0 100644
    a b class Manager(object):  
    137137    def create(self, **kwargs):
    138138        return self.get_query_set().create(**kwargs)
    139139
     140    def bulk_create(self, *args, **kwargs):
     141        return self.get_query_set().bulk_create(*args, **kwargs)
     142
    140143    def filter(self, *args, **kwargs):
    141144        return self.get_query_set().filter(*args, **kwargs)
    142145
    … … class Manager(object):  
    194197    def exists(self, *args, **kwargs):
    195198        return self.get_query_set().exists(*args, **kwargs)
    196199
    197     def _insert(self, values, **kwargs):
    198         return insert_query(self.model, values, **kwargs)
     200    def _insert(self, objs, fields, **kwargs):
     201        return insert_query(self.model, objs, fields, **kwargs)
    199202
    200203    def _update(self, values, **kwargs):
    201204        return self.get_query_set()._update(values, **kwargs)

  • django/db/models/query.py 

    diff --git a/django/db/models/query.py b/django/db/models/query.py
index 6a6a829..86060cd 100644
    a b from itertools import izip  
    77
    88from django.db import connections, router, transaction, IntegrityError
    99from django.db.models.aggregates import Aggregate
    10 from django.db.models.fields import DateField
     10from django.db.models.fields import DateField, AutoField
    1111from django.db.models.query_utils import (Q, select_related_descend,
    1212    deferred_class_factory, InvalidQuery)
    1313from django.db.models.deletion import Collector
    … … class QuerySet(object):  
    360360        obj.save(force_insert=True, using=self.db)
    361361        return obj
    362362
     363    def bulk_create(self, objs):
     364        """
     365        Inserts each of the instances into the database. This does *not* call
     366        save() on each of the instances, does not send any pre/post save
     367        signals, and does not set the primary key attribute if it is an
     368        autoincrement field.
     369        """
     370        # So this case is fun. When you bulk insert you don't get the primary
     371        # keys back (if it's an autoincrement), so you can't insert into the
     372        # child tables which references this. There are two workarounds, 1)
     373        # this could be implemented if you didn't have an autoincrement pk,
     374        # and 2) you could do it by doing O(n) normal inserts into the parent
     375        # tables to get the primary keys back, and then doing a single bulk
     376        # insert into the childmost table. We're punting on these for now
     377        # because they are relatively rare cases.
     378        if self.model._meta.parents:
     379            raise ValueError("Can't bulk create an inherited model")
     380        if not objs:
     381            return
     382        self._for_write = True
     383        connection = connections[self.db]
     384        fields = self.model._meta.local_fields
     385        if (connection.features.can_combine_inserts_with_and_without_auto_increment_pk
     386            and self.model._meta.has_auto_field):
     387            self.model._base_manager._insert(objs, fields=fields, using=self.db)
     388        else:
     389            objs_with_pk = [o for o in objs if o.pk]
     390            objs_without_pk = [o for o in objs if not o.pk]
     391            if objs_with_pk:
     392                self.model._base_manager._insert(objs_with_pk, fields=fields, using=self.db)
     393            if objs_without_pk:
     394                self.model._base_manager._insert(objs_without_pk, fields=[f for f in fields if not isinstance(f, AutoField)], using=self.db)
     395
    363396    def get_or_create(self, **kwargs):
    364397        """
    365398        Looks up an object with the given kwargs, creating one if necessary.
    … … class RawQuerySet(object):  
    14451478                self._model_fields[converter(column)] = field
    14461479        return self._model_fields
    14471480
    1448 def insert_query(model, values, return_id=False, raw_values=False, using=None):
     1481def insert_query(model, objs, fields, return_id=False, raw=False, using=None):
    14491482    """
    14501483    Inserts a new record for the given model. This provides an interface to
    14511484    the InsertQuery class and is how Model.save() is implemented. It is not
    14521485    part of the public API.
    14531486    """
    14541487    query = sql.InsertQuery(model)
    1455     query.insert_values(values, raw_values)
     1488    query.insert_values(fields, objs, raw=raw)
    14561489    return query.get_compiler(using=using).execute_sql(return_id)

  • django/db/models/sql/compiler.py 

    diff --git a/django/db/models/sql/compiler.py b/django/db/models/sql/compiler.py
index 841ec12..84dd7ed 100644
    a b  
     1from itertools import izip
     2
    13from django.core.exceptions import FieldError
    24from django.db import connections
    35from django.db import transaction
    … … from django.db.models.sql.query import get_proxied_model, get_order_dir, \  
    911     select_related_descend, Query
    1012from django.db.utils import DatabaseError
    1113
     14
    1215class SQLCompiler(object):
    1316    def __init__(self, query, connection, using):
    1417        self.query = query
    … … class SQLInsertCompiler(SQLCompiler):  
    794797        qn = self.connection.ops.quote_name
    795798        opts = self.query.model._meta
    796799        result = ['INSERT INTO %s' % qn(opts.db_table)]
    797         result.append('(%s)' % ', '.join([qn(c) for c in self.query.columns]))
    798         values = [self.placeholder(*v) for v in self.query.values]
    799         result.append('VALUES (%s)' % ', '.join(values))
    800         params = self.query.params
     800
     801        has_fields = bool(self.query.fields)
     802        fields = self.query.fields if has_fields else [opts.pk]
     803        result.append('(%s)' % ', '.join([qn(f.column) for f in fields]))
     804
     805        if has_fields:
     806            params = values = [
     807                [
     808                    f.get_db_prep_save(getattr(obj, f.attname) if self.query.raw else f.pre_save(obj, True), connection=self.connection)
     809                    for f in fields
     810                ]
     811                for obj in self.query.objs
     812            ]
     813        else:
     814            values = [[self.connection.ops.pk_default_value()] for obj in self.query.objs]
     815            params = [[]]
     816            fields = [None]
     817        can_bulk = not any(hasattr(field, "get_placeholder") for field in fields) and not self.return_id
     818
     819        if can_bulk:
     820            placeholders = [["%s"] * len(fields)]
     821        else:
     822            placeholders = [
     823                [self.placeholder(field, v) for field, v in izip(fields, val)]
     824                for val in values
     825            ]
    801826        if self.return_id and self.connection.features.can_return_id_from_insert:
    802             col = "%s.%s" % (qn(opts.db_table), qn(opts.pk.column))
     827            params = values[0]
     828            col = "%s.%s" % (qn(opst.db_table), qn(opts.pk.column))
     829            result.append("VALUES (%s)" % ", ".join(placeholders[0]))
    803830            r_fmt, r_params = self.connection.ops.return_insert_id()
    804831            result.append(r_fmt % col)
    805             params = params + r_params
    806         return ' '.join(result), params
     832            params += r_params
     833            return [(" ".join(result), tuple(param))]
     834        if can_bulk and self.connection.features.has_bulk_insert:
     835            result.append(self.connection.ops.bulk_insert_sql(fields, len(values)))
     836            return [(" ".join(result), tuple([v for val in values for v in val]))]
     837        else:
     838            return [
     839                (" ".join(result + ["VALUES (%s)" % ", ".join(p)]), vals)
     840                for p, vals in izip(placeholders, params)
     841            ]
    807842
    808843    def execute_sql(self, return_id=False):
     844        assert not (return_id and len(self.query.objs) != 1)
    809845        self.return_id = return_id
    810         cursor = super(SQLInsertCompiler, self).execute_sql(None)
     846        cursor = self.connection.cursor()
     847        for sql, params in self.as_sql():
     848            cursor.execute(sql, params)
    811849        if not (return_id and cursor):
    812850            return
    813851        if self.connection.features.can_return_id_from_insert:

  • django/db/models/sql/query.py 

    diff --git a/django/db/models/sql/query.py b/django/db/models/sql/query.py
index 99663b6..12b13a7 100644
    a b all about the internals of models in order to get the information it needs.  
    88"""
    99
    1010import copy
    11 from django.utils.tree import Node
     11
    1212from django.utils.datastructures import SortedDict
    1313from django.utils.encoding import force_unicode
     14from django.utils.tree import Node
    1415from django.db import connections, DEFAULT_DB_ALIAS
    1516from django.db.models import signals
    1617from django.db.models.fields import FieldDoesNotExist

  • django/db/models/sql/subqueries.py 

    diff --git a/django/db/models/sql/subqueries.py b/django/db/models/sql/subqueries.py
index 003bf43..39cfc03 100644
    a b class InsertQuery(Query):  
    138138
    139139    def __init__(self, *args, **kwargs):
    140140        super(InsertQuery, self).__init__(*args, **kwargs)
    141         self.columns = []
    142         self.values = []
    143         self.params = ()
     141        self.fields = []
     142        self.objs = []
    144143
    145144    def clone(self, klass=None, **kwargs):
    146145        extras = {
    147             'columns': self.columns[:],
    148             'values': self.values[:],
    149             'params': self.params
     146            'fields': self.fields[:],
     147            'objs': self.objs[:],
     148            'raw': self.raw,
    150149        }
    151150        extras.update(kwargs)
    152151        return super(InsertQuery, self).clone(klass, **extras)
    153152
    154     def insert_values(self, insert_values, raw_values=False):
     153    def insert_values(self, fields, objs, raw=False):
    155154        """
    156155        Set up the insert query from the 'insert_values' dictionary. The
    157156        dictionary gives the model field names and their target values.
    … … class InsertQuery(Query):  
    161160        parameters. This provides a way to insert NULL and DEFAULT keywords
    162161        into the query, for example.
    163162        """
    164         placeholders, values = [], []
    165         for field, val in insert_values:
    166             placeholders.append((field, val))
    167             self.columns.append(field.column)
    168             values.append(val)
    169         if raw_values:
    170             self.values.extend([(None, v) for v in values])
    171         else:
    172             self.params += tuple(values)
    173             self.values.extend(placeholders)
     163        self.fields = fields
     164        self.objs = objs
     165        self.raw = raw
    174166
    175167class DateQuery(Query):
    176168    """

  • docs/ref/models/querysets.txt 

    diff --git a/docs/ref/models/querysets.txt b/docs/ref/models/querysets.txt
index 2bd813d..b7a84cd 100644
    a b Though you usually won't create one manually -- you'll go through a  
    139139        clause or a default ordering on the model. ``False`` otherwise.
    140140
    141141    .. attribute:: db
    142    
     142
    143143        The database that will be used if this query is executed now.
    144144
    145145    .. note::
    … … has a side effect on your data. For more, see `Safe methods`_ in the HTTP spec.  
    11391139
    11401140.. _Safe methods: http://www.w3.org/Protocols/rfc2616/rfc2616-sec9.html#sec9.1.1
    11411141
     1142bulk_create
     1143~~~~~~~~~~~
     1144
     1145.. method:: bulk_create(objs)
     1146
     1147This method inserts the provided list of objects into the database in an
     1148efficient manner (generally only 1 query, no matter how many objects there
     1149are)::
     1150
     1151    >>> Entry.objects.bulk_create([
     1152    ...     Entry(headline="Django 1.0 Released"),
     1153    ...     Entry(headline="Django 1.1 Announced"),
     1154    ...     Entry(headline="Breaking: Django is awesome")
     1155    ... ])
     1156
     1157This has a number of caveats though:
     1158
     1159  * The model's ``save()`` method will not be called, and the ``pre_save`` and
     1160    ``post_save`` signals will not be sent.
     1161  * It does not work with child models in a multi-table inheritance scenario.
     1162  * If the model's primary key is an :class:`AutoField` it does not retrieve
     1163    and set the primary key attribute, as ``save()`` does.
     1164
    11421165count
    11431166~~~~~
    11441167

  • docs/topics/db/optimization.txt 

    diff --git a/docs/topics/db/optimization.txt b/docs/topics/db/optimization.txt
index 265ef55..24830aa 100644
    a b instead of::  
    268268
    269269   entry.blog.id
    270270
     271Insert in bulk
     272==============
     273
     274When creating objects, where possible, use the :meth:`QuerySet.bulk_create()`
     275method to reduce the number of SQL queries. For example::
     276
     277    Entry.objects.bulk_create([
     278        Entry(headline="Python 3.0 Released"),
     279        Entry(headline="Python 3.1 Planned")
     280    ])
     281
     282Is preferable to::
     283
     284    Entry.objects.create(headline="Python 3.0 Released")
     285    Entry.objects.create(headline="Python 3.1 Planned")
     286
     287This also applies to :class:`ManyToManyFields`, doing::
     288
     289    my_band.members.add(me, my_friend)
     290
     291Is preferable to::
     292
     293    my_band.members.add(me)
     294    my_band.members.add(my_friend)
     295
     296Where ``Bands`` and ``Artists`` have a many-to-many relationship.
     297 No newline at end of file

  • tests/regressiontests/bulk_create/models.py 

    diff --git a/tests/regressiontests/bulk_create/models.py b/tests/regressiontests/bulk_create/models.py
index 1f98a40..a4c611d 100644
    a b from django.db import models  
    33
    44class Country(models.Model):
    55    name = models.CharField(max_length=255)
    6     iso_two_letter = models.CharField(max_length=2)
    7  No newline at end of file
     6    iso_two_letter = models.CharField(max_length=2)
     7
     8class Place(models.Model):
     9    name = models.CharField(max_length=100)
     10
     11    class Meta:
     12        abstract = True
     13
     14class Restaurant(Place):
     15    pass
     16
     17class Pizzeria(Restaurant):
     18    pass
     19
     20class State(models.Model):
     21    two_letter_code = models.CharField(max_length=2, primary_key=True)
     22 No newline at end of file

  • tests/regressiontests/bulk_create/tests.py 

    diff --git a/tests/regressiontests/bulk_create/tests.py b/tests/regressiontests/bulk_create/tests.py
index 42ba095..020841c 100644
    a b  
     1from __future__ import with_statement
     2
    13from operator import attrgetter
    24
    35from django.test import TestCase, skipUnlessDBFeature
    46
    5 from models import Country
     7from models import Country, Restaurant, Pizzeria, State
    68
    79
    810class BulkCreateTests(TestCase):
    … … class BulkCreateTests(TestCase):  
    2325    @skipUnlessDBFeature("has_bulk_insert")
    2426    def test_efficiency(self):
    2527        with self.assertNumQueries(1):
    26             Country.objects.bulk_create(self.data)
    27  No newline at end of file
     28            Country.objects.bulk_create(self.data)
     29
     30    def test_inheritance(self):
     31        Restaurant.objects.bulk_create([
     32            Restaurant(name="Nicholas's")
     33        ])
     34        self.assertQuerysetEqual(Restaurant.objects.all(), [
     35            "Nicholas's",
     36        ], attrgetter("name"))
     37        with self.assertRaises(ValueError):
     38            Pizzeria.objects.bulk_create([
     39                Pizzeria(name="The Art of Pizza")
     40            ])
     41        self.assertQuerysetEqual(Pizzeria.objects.all(), [])
     42        self.assertQuerysetEqual(Restaurant.objects.all(), [
     43            "Nicholas's",
     44        ], attrgetter("name"))
     45
     46    def test_non_auto_increment_pk(self):
     47        State.objects.bulk_create([
     48            State(two_letter_code=s)
     49            for s in ["IL", "NY", "CA", "ME"]
     50        ])
     51        self.assertQuerysetEqual(State.objects.order_by("two_letter_code"), [
     52            "CA", "IL", "ME", "NY",
     53        ], attrgetter("two_letter_code"))
     54 No newline at end of file

  • tests/regressiontests/db_typecasts/tests.py 

    diff --git a/tests/regressiontests/db_typecasts/tests.py b/tests/regressiontests/db_typecasts/tests.py
index 8c71c8f..1d3bbfa 100644
    a b TEST_CASES = {  
    5353
    5454class DBTypeCasts(unittest.TestCase):
    5555    def test_typeCasts(self):
    56         for k, v in TEST_CASES.items():
     56        for k, v in TEST_CASES.iteritems():
    5757            for inpt, expected in v:
    5858                got = getattr(typecasts, k)(inpt)
    59                 assert got == expected, "In %s: %r doesn't match %r. Got %r instead." % (k, inpt, expected, got)
     59                self.assertEqual(got, expected, "In %s: %r doesn't match %r. Got %r instead." % (k, inpt, expected, got))
    6060
    6161if __name__ == '__main__':
    6262    unittest.main()
Back to Top