Ticket #1219: bulk-delete-v2.patch

django/db/models/base.py

@@ -3 +3 @@
-from django.db.models.fields import AutoField, ImageField
-from django.db.models.fields.related import OneToOne, ManyToOne
-from django.db.models.related import RelatedObject
-
+, delete_objects
-from django.db.models.options import Options, AdminOptions
-from django.db import connection, backend
-from django.db.models import signals
@@ -49 +49 @@
-        register_models(new_class._meta.app_label, new_class)
-        return new_class
-
-def cmp_cls(x, y):
-    for field in x._meta.fields:
-        if field.rel and not field.null and field.rel.to == y:
-            return -1
-    for field in y._meta.fields:
-        if field.rel and not field.null and field.rel.to == x:
-            return 1
-    return 0
-
-class Model(object):
-    __metaclass__ = ModelBase
-
@@ -187 +178 @@
-
-    save.alters_data = True
-
-_
+
-        """
-        Recursively populates seen_objs with all objects related to this object.
-        When done, seen_objs will be in the format:
@@ -207 +198 @@
-                except ObjectDoesNotExist:
-                    pass
-                else:
-_
+
-            else:
-                for sub_obj in getattr(self, rel_opts_name).all():
-_
+
-
-    def delete(self):
-        assert self._get_pk_val() is not None, "%s object can't be deleted because its %s attribute is set to None." % (self._meta.object_name, self._meta.pk.attname)
+        
+        # Find all the objects than need to be deleted
-        seen_objs = {}
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
-    delete.alters_data = True
-
-    def _get_FIELD_display(self, field):

django/db/models/manager.py

@@ -57 +57 @@
-    def dates(self, *args, **kwargs):
-        return self.get_query_set().dates(*args, **kwargs)
-
-    def delete(self, *args, **kwargs):
-        return self.get_query_set().delete(*args, **kwargs)
-
-    def distinct(self, *args, **kwargs):
-        return self.get_query_set().distinct(*args, **kwargs)
-

django/db/models/query.py

@@ -1 +1 @@
-from django.db import backend, connection
-from django.db.models.fields import DateField, FieldDoesNotExist
+from django.db.models import signals
+from django.dispatch import dispatcher
-from django.utils.datastructures import SortedDict
+
-import operator
-
-LOOKUP_SEPARATOR = '__'
@@ -125 +128 @@
-        extra_select = self._select.items()
-
-        cursor = connection.cursor()
-True
+
-        cursor.execute("SELECT " + (self._distinct and "DISTINCT " or "") + ",".join(select) + sql, params)
-        fill_cache = self._select_related
-        index_end = len(self.model._meta.fields)
@@ -149 +152 @@
-        counter._offset = None
-        counter._limit = None
-        counter._select_related = False
-True
+
-        cursor = connection.cursor()
-        cursor.execute("SELECT COUNT(*)" + sql, params)
-        return cursor.fetchone()[0]
@@ -171 +174 @@
-        assert bool(latest_by), "latest() requires either a field_name parameter or 'get_latest_by' in the model"
-        return self._clone(_limit=1, _order_by=('-'+latest_by,)).get()
-
-, *args, **kwargs
+
-        """
-
-
+
-        """
-
-
+
-
-        # Check for at least one query argument.
-        if not kwargs and not delete_all:
-            raise TypeError, "SAFETY MECHANISM: Specify DELETE_ALL=True if you actually want to delete all data."
-
-        if kwargs:
-            del_query = self.filter(*args, **kwargs)
-        else:
-            del_query = self._clone()
-        # disable non-supported fields
-        del_query._select_related = False
-        del_query._select = {}
-        del_query._order_by = []
-        del_query._offset = None
-        del_query._limit = None
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
-    ##################################################
-    # PUBLIC METHODS THAT RETURN A QUERYSET SUBCLASS #
-    ##################################################
@@ -297 +297 @@
-            self._result_cache = list(self.iterator())
-        return self._result_cache
-
-, allow_joins
+
-        opts = self.model._meta
-
-        # Construct the fundamental parts of the query: SELECT X FROM Y WHERE Z.
@@ -325 +325 @@
-        # Start composing the body of the SQL statement.
-        sql = [" FROM", backend.quote_name(opts.db_table)]
-
-        # Check if extra tables are allowed. If not, throw an error
-        if (tables or joins) and not allow_joins:
-            raise TypeError, "Joins are not allowed in this type of query"
-
-        # Compose the join dictionary into SQL describing the joins.
-        if joins:
-            sql.append(" ".join(["%s %s AS %s ON %s" % (join_type, table, alias, condition)
@@ -407 +403 @@
-            field_names = [f.attname for f in self.model._meta.fields]
-
-        cursor = connection.cursor()
-True
+
-        select = ['%s.%s' % (backend.quote_name(self.model._meta.db_table), backend.quote_name(c)) for c in columns]
-        cursor.execute("SELECT " + (self._distinct and "DISTINCT " or "") + ",".join(select) + sql, params)
-        while 1:
@@ -429 +425 @@
-        if self._field.null:
-            date_query._where.append('%s.%s IS NOT NULL' % \
-                (backend.quote_name(self.model._meta.db_table), backend.quote_name(self._field.column)))
-True
+
-        sql = 'SELECT %s %s GROUP BY 1 ORDER BY 1 %s' % \
-            (backend.get_date_trunc_sql(self._kind, '%s.%s' % (backend.quote_name(self.model._meta.db_table),
-            backend.quote_name(self._field.column))), sql, self._order)
@@ -762 +758 @@
-        params.extend(field.get_db_prep_lookup(clause, value))
-
-    return tables, joins, where, params
+
+def compare_models(x, y):
+    "Comparator for Models that puts models in an order where dependencies are easily resolved."
+    for field in x._meta.fields:
+        if field.rel and not field.null and field.rel.to == y:
+            return -1
+    for field in y._meta.fields:
+        if field.rel and not field.null and field.rel.to == x:
+            return 1
+    return 0
+
+def delete_objects(seen_objs):
+    "Iterate through a list of seen classes, and remove any instances that are referred to"
+    seen_classes = set(seen_objs.keys())
+    ordered_classes = list(seen_classes)
+    ordered_classes.sort(compare_models)
+
+    cursor = connection.cursor()
+     
+    for cls in ordered_classes:
+        seen_objs[cls] = seen_objs[cls].items()
+        seen_objs[cls].sort()
+    
+        # Pre notify all instances to be deleted
+        for pk_val, instance in seen_objs[cls]:
+            dispatcher.send(signal=signals.pre_delete, sender=cls, instance=instance)
+
+        pk_list = [pk for pk,instance in seen_objs[cls]]
+        for related in cls._meta.get_all_related_many_to_many_objects():
+            cursor.execute("DELETE FROM %s WHERE %s IN (%s)" % \
+                (backend.quote_name(related.field.get_m2m_db_table(related.opts)),
+                    backend.quote_name(cls._meta.object_name.lower() + '_id'),
+                    ','.join('%s' for pk in pk_list)), 
+                pk_list)
+        for f in cls._meta.many_to_many:
+            cursor.execute("DELETE FROM %s WHERE %s IN (%s)" % \
+                (backend.quote_name(f.get_m2m_db_table(cls._meta)),
+                    backend.quote_name(cls._meta.object_name.lower() + '_id'),
+                    ','.join(['%s' for pk in pk_list])), 
+                pk_list)
+        for field in cls._meta.fields:
+            if field.rel and field.null and field.rel.to in seen_classes:
+                cursor.execute("UPDATE %s SET %s=NULL WHERE %s IN (%s)" % \
+                    (backend.quote_name(cls._meta.db_table), 
+                        backend.quote_name(field.column),
+                        backend.quote_name(cls._meta.pk.column), 
+                        ','.join(['%s' for pk in pk_list])), 
+                    pk_list)
+
+    # Now delete the actual data
+    for cls in ordered_classes:
+        seen_objs[cls].reverse()
+        pk_list = [pk for pk,instance in seen_objs[cls]]
+        
+        cursor.execute("DELETE FROM %s WHERE %s IN (%s)" % \
+            (backend.quote_name(cls._meta.db_table), 
+                backend.quote_name(cls._meta.pk.column),
+                ','.join(['%s' for pk in pk_list])),
+            pk_list)
+                
+        # Last cleanup; set NULLs where there once was a reference to the object,
+        # NULL the primary key of the found objects, and perform post-notification.
+        for pk_val, instance in seen_objs[cls]:
+            for field in cls._meta.fields:
+                if field.rel and field.null and field.rel.to in seen_classes:
+                    setattr(instance, field.attname, None)
+
+            setattr(instance, cls._meta.pk.attname, None)
+            dispatcher.send(signal=signals.post_delete, sender=cls, instance=instance)
+
+    connection.commit()

tests/modeltests/basic/models.py

@@ -9 +9 @@
-class Article(models.Model):
-    headline = models.CharField(maxlength=100, default='Default headline')
-    pub_date = models.DateTimeField()
-
+
+
+
-API_TESTS = """
-
-# No articles are in the system yet.
@@ -37 +39 @@
->>> a.headline = 'Area woman programs in Python'
->>> a.save()
-
-
-
-
+
->>> Article.objects.all()
-<Article object>
+Area woman programs in Python
-
-# Django provides a rich database lookup API.
->>> Article.objects.get(id__exact=1)
-
+
->>> Article.objects.get(headline__startswith='Area woman')
-
+
->>> Article.objects.get(pub_date__year=2005)
-
+
->>> Article.objects.get(pub_date__year=2005, pub_date__month=7)
-
+
->>> Article.objects.get(pub_date__year=2005, pub_date__month=7, pub_date__day=28)
-
+
-
-# The "__exact" lookup type can be omitted, as a shortcut.
->>> Article.objects.get(id=1)
-
+
->>> Article.objects.get(headline='Area woman programs in Python')
-
+
-
->>> Article.objects.filter(pub_date__year=2005)
-<Article object>
+Area woman programs in Python
->>> Article.objects.filter(pub_date__year=2004)
-[]
->>> Article.objects.filter(pub_date__year=2005, pub_date__month=7)
-<Article object>
+Area woman programs in Python
-
-# Django raises an Article.DoesNotExist exception for get() if the parameters
-# don't match any object.
@@ -84 +84 @@
-# shortcut for primary-key exact lookups.
-# The following is identical to articles.get(id=1).
->>> Article.objects.get(pk=1)
-
+
-
-# Model instances of the same type and same ID are considered equal.
->>> a = Article.objects.get(pk=1)
@@ -234 +234 @@
-
-# You can get items using index and slice notation.
->>> Article.objects.all()[0]
-
+
->>> Article.objects.all()[1:3]
-<Article object>, <Article object>
+Second article, Third article
->>> s3 = Article.objects.filter(id__exact=3)
->>> (s1 | s2 | s3)[::2]
-<Article object>, <Article object>
+Area woman programs in Python, Third article
-
-# An Article instance doesn't have access to the "objects" attribute.
-# That's only available on the class.
@@ -254 +254 @@
-AttributeError: Manager isn't accessible via Article instances
-
-# Bulk delete test: How many objects before and after the delete?
-count
-
-delete(id__lte=4
-count
-
+all
+
+filter(id__lte=4).delete(
+all
+
-
->>> Article.objects.delete()
-Traceback (most recent call last):
-    ...
-TypeError: SAFETY MECHANISM: Specify DELETE_ALL=True if you actually want to delete all data.
-
->>> Article.objects.delete(DELETE_ALL=True)
->>> Article.objects.count()
-0L
-
-"""
-
-from django.conf import settings

tests/modeltests/many_to_many/models.py

@@ -162 +162 @@
->>> p2.article_set.all().order_by('headline')
-[Oxygen-free diet works wonders]
-
+# Recreate the article and Publication we just deleted.
+>>> p1 = Publication(id=None, title='The Python Journal')
+>>> p1.save()
+>>> a2 = Article(id=None, headline='NASA uses Python')
+>>> a2.save()
+>>> a2.publications.add(p1, p2, p3)
-
+# Bulk delete some Publications - references to deleted publications should go
+>>> Publication.objects.filter(title__startswith='Science').delete()
+>>> Publication.objects.all()
+[Highlights for Children, The Python Journal]
+>>> Article.objects.all()
+[Django lets you build Web apps easily, NASA finds intelligent life on Earth, Oxygen-free diet works wonders, NASA uses Python]
+>>> a2.publications.all()
+[The Python Journal]
-
+# Bulk delete some articles - references to deleted objects should go
+>>> q = Article.objects.filter(headline__startswith='Django')
+>>> print q
+[Django lets you build Web apps easily]
+>>> q.delete()
+
+# After the delete, the QuerySet cache needs to be cleared, and the referenced objects should be gone
+>>> print q
+[]
+>>> p1.article_set.all()
+[NASA uses Python]
+
+
-"""

tests/modeltests/many_to_one/models.py

@@ -94 +94 @@
-
-# The underlying query only makes one join when a related table is referenced twice.
->>> query = Article.objects.filter(reporter__first_name__exact='John', reporter__last_name__exact='Smith')
-True
+
->>> sql.count('INNER JOIN')
-1
-
@@ -163 +163 @@
->>> Reporter.objects.filter(article__reporter__first_name__startswith='John').distinct()
-[John Smith]
-
-# Deletes that require joins are prohibited.
->>> Article.objects.delete(reporter__first_name__startswith='Jo')
-Traceback (most recent call last):
-    ...
-TypeError: Joins are not allowed in this type of query
-
-# If you delete a reporter, his articles will be deleted.
->>> Article.objects.order_by('headline')
-[John's second story, Paul's story, This is a test, This is a test, This is a test]
->>> Reporter.objects.order_by('first_name')
-[John Smith, Paul Jones]
-
+2
->>> Article.objects.order_by('headline')
-Paul's story
+John's second story, This is a test, This is a test, This is a test
->>> Reporter.objects.order_by('first_name')
-Paul Jones
+John Smith
-
+# Deletes using a join in the query
+>>> Reporter.objects.filter(article__headline__startswith='This').delete()
+>>> Reporter.objects.all()
+[]
+>>> Article.objects.all()
+[]
+
-"""