QuerySet cloning can sometimes fail

[Dorian Grey <imgrey@…>]

[Original bug report removed; see ​the second comment for the original report. --JKM]

Under some circumstances -- often involving select_related() and count() -- QuerySet.clone() can cause exceptions related to deepcopy(). These exceptions look like TypeError: instancemethod expected at least 2 arguments, got 0 or TypeError: function expected at least 2 arguments, got 0.

The first one involving instancemethod is probably related to a Python issue: see ​http://bugs.python.org/issue1515.

The second involving function only seems to occur on Python 2.4 (and not 2.5), but since 1515 isn't fixed yet, it may be a seperate problem.

In either case, though, the root problem is that for some reason QuerySet is trying to clone instancemethods and unbound functions, which isn't exactly supported -- dunno what a copy of a function would do anyway. The likely fix will involve finding out where and why functions/methods are being coppied, and stop doing that.

[oyvind]

Post a simple example showing count not working.

[Dorian Grey <imgrey@…>]

complete moldels.py:

from django.db import models
from django.contrib.auth.models import User
from django.dispatch import dispatcher
from django.db.models import signals

def _add_attrs_and_methods(sender, instance, signal, *args, **kwargs):
    import types

    def can_change(self, post):
	if self == post.author:
	    return True
	return False

    instance.can_change = types.MethodType(can_change, instance, instance.__class__)

class Halb(models.Model):
    fk = models.ForeignKey(User)
    field = models.CharField(max_length=10)

dispatcher.connect(_add_attrs_and_methods, sender=User, signal=signals.post_init)

In [1]: from users.models import *

In [2]: u = User.objects.all()[0]

In [3]: Halb.objects.filter(fk=u).count()
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)

/tmp/tst/<ipython console> 

/usr/lib/python2.4/site-packages/django/db/models/query.py in count(self)
    182             return len(self._result_cache)
    183 
--> 184         return self.query.get_count()
    185 
    186     def get(self, *args, **kwargs):

/usr/lib/python2.4/site-packages/django/db/models/sql/query.py in get_count(self)
    209         """
    210         from subqueries import CountQuery
--> 211         obj = self.clone()
    212         obj.clear_ordering(True)
    213         obj.clear_limits()

/usr/lib/python2.4/site-packages/django/db/models/sql/query.py in clone(self, klass, **kwargs)
    167         obj.select = self.select[:]
    168         obj.tables = self.tables[:]
--> 169         obj.where = deepcopy(self.where)
    170         obj.where_class = self.where_class
    171         obj.group_by = self.group_by[:]

/usr/lib/python2.4/copy.py in deepcopy(x, memo, _nil)
    183             copier = _getspecial(cls, "__deepcopy__")
    184             if copier:
--> 185                 y = copier(x, memo)
    186             else:
    187                 reductor = dispatch_table.get(cls)

/usr/lib/python2.4/site-packages/django/utils/tree.py in __deepcopy__(self, memodict)
     43         obj = Node(connector=self.connector, negated=self.negated)
     44         obj.__class__ = self.__class__
---> 45         obj.children = deepcopy(self.children, memodict)
     46         obj.subtree_parents = deepcopy(self.subtree_parents, memodict)
     47         return obj

/usr/lib/python2.4/copy.py in deepcopy(x, memo, _nil)
    172     copier = _deepcopy_dispatch.get(cls)
    173     if copier:
--> 174         y = copier(x, memo)
    175     else:
    176         try:

/usr/lib/python2.4/copy.py in _deepcopy_list(x, memo)
    239     memo[id(x)] = y
    240     for a in x:
--> 241         y.append(deepcopy(a, memo))
    242     return y
    243 d[types.ListType] = _deepcopy_list

/usr/lib/python2.4/copy.py in deepcopy(x, memo, _nil)
    172     copier = _deepcopy_dispatch.get(cls)
    173     if copier:
--> 174         y = copier(x, memo)
    175     else:
    176         try:

/usr/lib/python2.4/copy.py in _deepcopy_tuple(x, memo)
    246     y = []
    247     for a in x:
--> 248         y.append(deepcopy(a, memo))
    249     d = id(x)
    250     try:

/usr/lib/python2.4/copy.py in deepcopy(x, memo, _nil)
    202                             raise Error(
    203                                 "un(deep)copyable object of type %s" % cls)
--> 204                 y = _reconstruct(x, rv, 1, memo)
    205 
    206     memo[d] = y

/usr/lib/python2.4/copy.py in _reconstruct(x, info, deep, memo)
    349     if state:
    350         if deep:
--> 351             state = deepcopy(state, memo)
    352         if hasattr(y, '__setstate__'):
    353             y.__setstate__(state)

/usr/lib/python2.4/copy.py in deepcopy(x, memo, _nil)
    172     copier = _deepcopy_dispatch.get(cls)
    173     if copier:
--> 174         y = copier(x, memo)
    175     else:
    176         try:

/usr/lib/python2.4/copy.py in _deepcopy_dict(x, memo)
    266     memo[id(x)] = y
    267     for key, value in x.iteritems():
--> 268         y[deepcopy(key, memo)] = deepcopy(value, memo)
    269     return y
    270 d[types.DictionaryType] = _deepcopy_dict

/usr/lib/python2.4/copy.py in deepcopy(x, memo, _nil)
    202                             raise Error(
    203                                 "un(deep)copyable object of type %s" % cls)
--> 204                 y = _reconstruct(x, rv, 1, memo)
    205 
    206     memo[d] = y

/usr/lib/python2.4/copy.py in _reconstruct(x, info, deep, memo)
    334     if deep:
    335         args = deepcopy(args, memo)
--> 336     y = callable(*args)
    337     memo[id(x)] = y
    338     if listiter is not None:

/usr/lib/python2.4/copy_reg.py in __newobj__(cls, *args)
     90 
     91 def __newobj__(cls, *args):
---> 92     return cls.__new__(cls, *args)
     93 
     94 def _slotnames(cls):

TypeError: instancemethod expected at least 2 arguments, got 0

[Andy McCurdy <sedrik@…>]

Running r7494 of Django, I'm seeing the same behavior on python 2.4. python 2.5 seems to work as expected.

[Jacob]

Changed title, description to more clearly explain what's going on here