Code

Ticket #7052: t7052-surrogate-key.2.diff

File t7052-surrogate-key.2.diff, 42.9 KB (added by russellm, 4 years ago)

Second draft of surrogate-key based serialization.

Line 
1diff -r f266dfb70b06 django/contrib/auth/models.py
2--- a/django/contrib/auth/models.py     Wed Nov 04 11:38:06 2009 +0000
3+++ b/django/contrib/auth/models.py     Sat Dec 05 01:25:01 2009 +0800
4@@ -47,6 +47,13 @@
5 class SiteProfileNotAvailable(Exception):
6     pass
7 
8+class PermissionManager(models.Manager):
9+    def get_by_surrogate(self, codename, app_label, model):
10+        return self.get(
11+            codename=codename,
12+            content_type=ContentType.objects.get_by_surrogate(app_label, model)
13+        )
14+
15 class Permission(models.Model):
16     """The permissions system provides a way to assign permissions to specific users and groups of users.
17 
18@@ -63,6 +70,7 @@
19     name = models.CharField(_('name'), max_length=50)
20     content_type = models.ForeignKey(ContentType)
21     codename = models.CharField(_('codename'), max_length=100)
22+    objects = PermissionManager()
23 
24     class Meta:
25         verbose_name = _('permission')
26@@ -76,6 +84,10 @@
27             unicode(self.content_type),
28             unicode(self.name))
29 
30+    def surrogate_key(self):
31+        return (self.codename,) + self.content_type.surrogate_key()
32+    surrogate_key.dependencies = ['contenttypes.contenttype']
33+
34 class Group(models.Model):
35     """Groups are a generic way of categorizing users to apply permissions, or some other label, to those users. A user can belong to any number of groups.
36 
37diff -r f266dfb70b06 django/contrib/contenttypes/models.py
38--- a/django/contrib/contenttypes/models.py     Wed Nov 04 11:38:06 2009 +0000
39+++ b/django/contrib/contenttypes/models.py     Sat Dec 05 01:25:01 2009 +0800
40@@ -8,6 +8,13 @@
41     # This cache is shared by all the get_for_* methods.
42     _cache = {}
43 
44+    def get_by_surrogate(self, app_label, model):
45+        try:
46+            ct = self.__class__._cache[(app_label, model)]
47+        except KeyError:
48+            ct = self.get(app_label=app_label, model=model)
49+        return ct
50+
51     def get_for_model(self, model):
52         """
53         Returns the ContentType object for a given model, creating the
54@@ -93,3 +100,6 @@
55         so code that calls this method should catch it.
56         """
57         return self.model_class()._default_manager.get(**kwargs)
58+
59+    def surrogate_key(self):
60+        return (self.app_label, self.model)
61diff -r f266dfb70b06 django/core/management/commands/dumpdata.py
62--- a/django/core/management/commands/dumpdata.py       Wed Nov 04 11:38:06 2009 +0000
63+++ b/django/core/management/commands/dumpdata.py       Sat Dec 05 01:25:01 2009 +0800
64@@ -67,14 +67,11 @@
65         except KeyError:
66             raise CommandError("Unknown serialization format: %s" % format)
67 
68+        # Now collate the objects to be serialized.
69         objects = []
70-        for app, model_list in app_list.items():
71-            if model_list is None:
72-                model_list = get_models(app)
73-
74-            for model in model_list:
75-                if not model._meta.proxy:
76-                    objects.extend(model._default_manager.all())
77+        for model in sort_dependencies(app_list.items()):
78+            if not model._meta.proxy:
79+                objects.extend(model._default_manager.all())
80 
81         try:
82             return serializers.serialize(format, objects, indent=indent)
83@@ -82,3 +79,60 @@
84             if show_traceback:
85                 raise
86             raise CommandError("Unable to serialize database: %s" % e)
87+
88+def sort_dependencies(app_list):
89+    """Sort a list of app,modellist pairs into a single list of models.
90+
91+    The single list of models is sorted so that any model with a surrogate key
92+    is serialized before a normal model, and any model with a surrogate key
93+    dependency has it's dependencies serialized first.
94+    """
95+    from django.db.models import get_model, get_models
96+    # Split the models to be serialized into two lists:
97+    # - the models that provide surrogates
98+    # - the models that are serialized as-is.
99+    dependent_models = []
100+    surrogate_model_list = []
101+    normal_model_list = []
102+    for app, model_list in app_list:
103+        if model_list is None:
104+            model_list = get_models(app)
105+
106+        for model in model_list:
107+            if hasattr(model, 'surrogate_key'):
108+                deps = getattr(model.surrogate_key, 'dependencies', None)
109+                if deps:
110+                    dependent_models.append((model, [get_model(*d.split('.')) for d in deps]))
111+                else:
112+                    surrogate_model_list.append(model)
113+            else:
114+                normal_model_list.append(model)
115+
116+    # Now sort the surrogates to ensure that dependencies are met.
117+    # As this process runs, items will be removed from the dependent_models
118+    # and put onto the end of the surrogate_model_list. Stop when either
119+    #  1) There are no dependent models left, or
120+    #  2) We've done a full iteration over the dependent models and
121+    #     haven't been able to add a model to the surrogate list. This
122+    #     indicates that there are circular dependencies
123+    while dependent_models:
124+        skipped = []
125+        changed = False
126+        while dependent_models:
127+            model, deps = dependent_models.pop()
128+            # If all of the models in the dependency list are already on the
129+            # surrogate model list, then we've found another model with all
130+            # it's dependencies satisfied.
131+            if all((d in surrogate_model_list) for d in deps):
132+                surrogate_model_list.append(model)
133+                changed = True
134+            else:
135+                skipped.append((model, deps))
136+        if not changed:
137+            raise CommandError("Can't resolve dependencies for %s in serialized app list." %
138+                ', '.join('%s.%s' % (model._meta.app_label, model._meta.object_name)
139+                for model, deps in skipped)
140+            )
141+        dependent_models = skipped
142+
143+    return surrogate_model_list + normal_model_list
144diff -r f266dfb70b06 django/core/serializers/python.py
145--- a/django/core/serializers/python.py Wed Nov 04 11:38:06 2009 +0000
146+++ b/django/core/serializers/python.py Sat Dec 05 01:25:01 2009 +0800
147@@ -47,17 +47,24 @@
148     def handle_fk_field(self, obj, field):
149         related = getattr(obj, field.name)
150         if related is not None:
151-            if field.rel.field_name == related._meta.pk.name:
152-                # Related to remote object via primary key
153-                related = related._get_pk_val()
154+            if hasattr(related, 'surrogate_key'):
155+                related = related.surrogate_key()
156             else:
157-                # Related to remote object via other field
158-                related = getattr(related, field.rel.field_name)
159-        self._current[field.name] = smart_unicode(related, strings_only=True)
160+                if field.rel.field_name == related._meta.pk.name:
161+                    # Related to remote object via primary key
162+                    related = related._get_pk_val()
163+                else:
164+                    # Related to remote object via other field
165+                    related = smart_unicode(getattr(related, field.rel.field_name), strings_only=True)
166+        self._current[field.name] = related
167 
168     def handle_m2m_field(self, obj, field):
169         if field.rel.through._meta.auto_created:
170-            self._current[field.name] = [smart_unicode(related._get_pk_val(), strings_only=True)
171+            if hasattr(field.rel.to, 'surrogate_key'):
172+                m2m_value = lambda value: value.surrogate_key()
173+            else:
174+                m2m_value = lambda value: smart_unicode(value._get_pk_val(), strings_only=True)
175+            self._current[field.name] = [m2m_value(related)
176                                for related in getattr(obj, field.name).iterator()]
177 
178     def getvalue(self):
179@@ -86,13 +93,28 @@
180 
181             # Handle M2M relations
182             if field.rel and isinstance(field.rel, models.ManyToManyRel):
183-                m2m_convert = field.rel.to._meta.pk.to_python
184-                m2m_data[field.name] = [m2m_convert(smart_unicode(pk)) for pk in field_value]
185+                if hasattr(field.rel.to._default_manager, 'get_by_surrogate'):
186+                    def m2m_convert(value):
187+                        if hasattr(value, '__iter__'):
188+                            obj = field.rel.to._default_manager.get_by_surrogate(*value)
189+                        else:
190+                            obj = field.rel.to._default_manager.get_by_surrogate(value)
191+                        return obj.pk
192+                else:
193+                    m2m_convert = lambda v: smart_unicode(field.rel.to._meta.pk.to_python(v))
194+                m2m_data[field.name] = [m2m_convert(pk) for pk in field_value]
195 
196             # Handle FK fields
197             elif field.rel and isinstance(field.rel, models.ManyToOneRel):
198                 if field_value is not None:
199-                    data[field.attname] = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value)
200+                    if hasattr(field.rel.to._default_manager, 'get_by_surrogate'):
201+                        if hasattr(field_value, '__iter__'):
202+                            obj = field.rel.to._default_manager.get_by_surrogate(*field_value)
203+                        else:
204+                            obj = field.rel.to._default_manager.get_by_surrogate(field_value)
205+                        data[field.attname] = getattr(obj, field.rel.field_name)
206+                    else:
207+                        data[field.attname] = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value)
208                 else:
209                     data[field.attname] = None
210 
211diff -r f266dfb70b06 django/core/serializers/xml_serializer.py
212--- a/django/core/serializers/xml_serializer.py Wed Nov 04 11:38:06 2009 +0000
213+++ b/django/core/serializers/xml_serializer.py Sat Dec 05 01:25:01 2009 +0800
214@@ -81,13 +81,26 @@
215         self._start_relational_field(field)
216         related = getattr(obj, field.name)
217         if related is not None:
218-            if field.rel.field_name == related._meta.pk.name:
219-                # Related to remote object via primary key
220-                related = related._get_pk_val()
221+            if hasattr(related, 'surrogate_key'):
222+                # If related object has a surrogate, use it
223+                related = related.surrogate_key()
224+                if hasattr(related, '__iter__'):
225+                    # Iterable surrogates are rolled out as subelements
226+                    for key_value in related:
227+                        self.xml.addQuickElement("surrogate", attrs={
228+                            'value' : smart_unicode(key_value)
229+                        })
230+                else:
231+                    # Otherwise just output the key value as a string
232+                    self.xml.characters(smart_unicode(related))
233             else:
234-                # Related to remote object via other field
235-                related = getattr(related, field.rel.field_name)
236-            self.xml.characters(smart_unicode(related))
237+                if field.rel.field_name == related._meta.pk.name:
238+                    # Related to remote object via primary key
239+                    related = related._get_pk_val()
240+                else:
241+                    # Related to remote object via other field
242+                    related = getattr(related, field.rel.field_name)
243+                self.xml.characters(smart_unicode(related))
244         else:
245             self.xml.addQuickElement("None")
246         self.xml.endElement("field")
247@@ -100,8 +113,31 @@
248         """
249         if field.rel.through._meta.auto_created:
250             self._start_relational_field(field)
251+            if hasattr(field.rel.to, 'surrogate_key'):
252+                # If the objects in the m2m have a surrogate, use it
253+                def handle_m2m(value):
254+                    surrogate = value.surrogate_key()
255+                    if hasattr(surrogate, '__iter__'):
256+                        # Iterable surrogates are rolled out as subelements
257+                        self.xml.startElement("object", {})
258+                        for key_value in surrogate:
259+                            self.xml.addQuickElement("surrogate", attrs={
260+                                'value' : smart_unicode(key_value)
261+                            })
262+                        self.xml.endElement("object")
263+                    else:
264+                        # Otherwise just output the element
265+                        self.xml.addQuickElement("object", attrs={
266+                            'surrogate' : smart_unicode(surrogate)
267+                        })
268+            else:
269+                def handle_m2m(value):
270+                    self.xml.addQuickElement("object", attrs={
271+                        'pk' : smart_unicode(value._get_pk_val())
272+                    })
273             for relobj in getattr(obj, field.name).iterator():
274-                self.xml.addQuickElement("object", attrs={"pk" : smart_unicode(relobj._get_pk_val())})
275+                handle_m2m(relobj)
276+
277             self.xml.endElement("field")
278 
279     def _start_relational_field(self, field):
280@@ -187,16 +223,40 @@
281         if node.getElementsByTagName('None'):
282             return None
283         else:
284-            return field.rel.to._meta.get_field(field.rel.field_name).to_python(
285-                       getInnerText(node).strip())
286+            if hasattr(field.rel.to._default_manager, 'get_by_surrogate'):
287+                keys = node.getElementsByTagName('surrogate')
288+                if keys:
289+                    # If there are 'key' subelements, it must be an iterable surrogate
290+                    field_value = [k.getAttribute('value') for k in keys]
291+                    obj = field.rel.to._default_manager.get_by_surrogate(*field_value)
292+                else:
293+                    # Otherwise, just use the field text as the surrogate
294+                    field_value = getInnerText(node).strip()
295+                    obj = field.rel.to._default_manager.get_by_surrogate(field_value)
296+                return getattr(obj, field.rel.field_name)
297+            else:
298+                field_value = getInnerText(node).strip()
299+                return field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value)
300 
301     def _handle_m2m_field_node(self, node, field):
302         """
303         Handle a <field> node for a ManyToManyField.
304         """
305-        return [field.rel.to._meta.pk.to_python(
306-                    c.getAttribute("pk"))
307-                    for c in node.getElementsByTagName("object")]
308+        if hasattr(field.rel.to._default_manager, 'get_by_surrogate'):
309+            def m2m_convert(n):
310+                keys = n.getElementsByTagName('surrogate')
311+                if keys:
312+                    # If there are 'key' subelements, it must be an iterable surrogate
313+                    field_value = [k.getAttribute('value') for k in keys]
314+                    obj = field.rel.to._default_manager.get_by_surrogate(*field_value)
315+                else:
316+                    # Otherwise, just use the field attribute as the surrogate
317+                    field_value = n.getAttribute('surrogate')
318+                    obj = field.rel.to._default_manager.get_by_surrogate(field_value)
319+                return obj.pk
320+        else:
321+            m2m_convert = lambda n: field.rel.to._meta.pk.to_python(n.getAttribute('pk'))
322+        return [m2m_convert(c) for c in node.getElementsByTagName("object")]
323 
324     def _get_model_from_node(self, node, attr):
325         """
326diff -r f266dfb70b06 tests/modeltests/fixtures/fixtures/fixture6.json
327--- /dev/null   Thu Jan 01 00:00:00 1970 +0000
328+++ b/tests/modeltests/fixtures/fixtures/fixture6.json  Sat Dec 05 01:25:01 2009 +0800
329@@ -0,0 +1,41 @@
330+[
331+    {
332+        "pk": "1",
333+        "model": "fixtures.tag",
334+        "fields": {
335+            "name": "copyright",
336+            "tagged_type": ["fixtures", "article"],
337+            "tagged_id": "3"
338+        }
339+    },
340+    {
341+        "pk": "2",
342+        "model": "fixtures.tag",
343+        "fields": {
344+            "name": "law",
345+            "tagged_type": ["fixtures", "article"],
346+            "tagged_id": "3"
347+        }
348+    },
349+    {
350+       "pk": "1",
351+       "model": "fixtures.person",
352+       "fields": {
353+           "name": "Django Reinhardt"
354+       }
355+    },
356+    {
357+       "pk": "2",
358+       "model": "fixtures.person",
359+       "fields": {
360+           "name": "Stephane Grappelli"
361+       }
362+    },
363+        {
364+       "pk": "3",
365+       "model": "fixtures.person",
366+       "fields": {
367+           "name": "Prince"
368+       }
369+    }
370+]
371diff -r f266dfb70b06 tests/modeltests/fixtures/fixtures/fixture7.xml
372--- /dev/null   Thu Jan 01 00:00:00 1970 +0000
373+++ b/tests/modeltests/fixtures/fixtures/fixture7.xml   Sat Dec 05 01:25:01 2009 +0800
374@@ -0,0 +1,27 @@
375+<?xml version="1.0" encoding="utf-8"?>
376+<django-objects version="1.0">
377+    <object pk="2" model="fixtures.tag">
378+        <field type="CharField" name="name">legal</field>
379+        <field to="contenttypes.contenttype" name="tagged_type" rel="ManyToOneRel">
380+            <surrogate value="fixtures"/>
381+            <surrogate value="article"/>
382+        </field>
383+        <field type="PositiveIntegerField" name="tagged_id">3</field>
384+    </object>
385+    <object pk="3" model="fixtures.tag">
386+        <field type="CharField" name="name">django</field>
387+        <field to="contenttypes.contenttype" name="tagged_type" rel="ManyToOneRel">
388+            <surrogate value="fixtures"/>
389+            <surrogate value="article"/>
390+        </field>
391+        <field type="PositiveIntegerField" name="tagged_id">4</field>
392+    </object>
393+    <object pk="4" model="fixtures.tag">
394+        <field type="CharField" name="name">world domination</field>
395+        <field to="contenttypes.contenttype" name="tagged_type" rel="ManyToOneRel">
396+            <surrogate value="fixtures"/>
397+            <surrogate value="article"/>
398+        </field>
399+        <field type="PositiveIntegerField" name="tagged_id">4</field>
400+    </object>
401+</django-objects>
402diff -r f266dfb70b06 tests/modeltests/fixtures/fixtures/fixture8.json
403--- /dev/null   Thu Jan 01 00:00:00 1970 +0000
404+++ b/tests/modeltests/fixtures/fixtures/fixture8.json  Sat Dec 05 01:25:01 2009 +0800
405@@ -0,0 +1,32 @@
406+[
407+    {
408+       "pk": "1",
409+       "model": "fixtures.visa",
410+       "fields": {
411+           "person": "Django Reinhardt",
412+           "permissions": [
413+               ["add_user", "auth", "user"],
414+               ["change_user", "auth", "user"],
415+               ["delete_user", "auth", "user"]
416+           ]
417+       }
418+    },
419+    {
420+       "pk": "2",
421+       "model": "fixtures.visa",
422+       "fields": {
423+           "person": "Stephane Grappelli",
424+           "permissions": [
425+               ["add_user", "auth", "user"]
426+           ]
427+       }
428+    },
429+        {
430+       "pk": "3",
431+       "model": "fixtures.visa",
432+       "fields": {
433+           "person": "Prince",
434+           "permissions": []
435+       }
436+    }
437+]
438diff -r f266dfb70b06 tests/modeltests/fixtures/fixtures/fixture9.xml
439--- /dev/null   Thu Jan 01 00:00:00 1970 +0000
440+++ b/tests/modeltests/fixtures/fixtures/fixture9.xml   Sat Dec 05 01:25:01 2009 +0800
441@@ -0,0 +1,38 @@
442+<?xml version="1.0" encoding="utf-8"?>
443+<django-objects version="1.0">
444+    <object pk="2" model="fixtures.visa">
445+        <field type="CharField" name="person">Stephane Grappelli</field>
446+        <field to="auth.permission" name="permissions" rel="ManyToManyRel">
447+            <object>
448+                <surrogate value="add_user"/>
449+                <surrogate value="auth"/>
450+                <surrogate value="user"/>
451+            </object>
452+            <object>
453+                <surrogate value="delete_user"/>
454+                <surrogate value="auth"/>
455+                <surrogate value="user"/>
456+            </object>
457+        </field>
458+    </object>
459+    <object pk="3" model="fixtures.person">
460+        <field type="CharField" name="name">Artist formerly known as &quot;Prince&quot;</field>
461+    </object>
462+    <object pk="3" model="fixtures.visa">
463+        <field type="CharField" name="person">Artist formerly known as &quot;Prince&quot;</field>
464+        <field to="auth.permission" name="permissions" rel="ManyToManyRel">
465+            <object>
466+                <surrogate value="change_user"/>
467+                <surrogate value="auth"/>
468+                <surrogate value="user"/>
469+            </object>
470+        </field>
471+    </object>
472+    <object pk="1" model="fixtures.book">
473+        <field type="CharField" name="name">Music for all ages</field>
474+        <field to="fixtures.person" name="authors" rel="ManyToManyRel">
475+            <object surrogate="Django Reinhardt"/>
476+            <object surrogate="Artist formerly known as &quot;Prince&quot;"/>
477+        </field>
478+    </object>
479+</django-objects>
480diff -r f266dfb70b06 tests/modeltests/fixtures/models.py
481--- a/tests/modeltests/fixtures/models.py       Wed Nov 04 11:38:06 2009 +0000
482+++ b/tests/modeltests/fixtures/models.py       Sat Dec 05 01:25:01 2009 +0800
483@@ -8,6 +8,9 @@
484 ``FIXTURE_DIRS`` setting.
485 """
486 
487+from django.contrib.auth.models import Permission
488+from django.contrib.contenttypes import generic
489+from django.contrib.contenttypes.models import ContentType
490 from django.db import models
491 from django.conf import settings
492 
493@@ -31,6 +34,62 @@
494     class Meta:
495         ordering = ('-pub_date', 'headline')
496 
497+class Blog(models.Model):
498+    name = models.CharField(max_length=100)
499+    featured = models.ForeignKey(Article, related_name='fixtures_featured_set')
500+    articles = models.ManyToManyField(Article, blank=True,
501+                                      related_name='fixtures_articles_set')
502+
503+    def __unicode__(self):
504+        return self.name
505+
506+
507+class Tag(models.Model):
508+    name = models.CharField(max_length=100)
509+    tagged_type = models.ForeignKey(ContentType, related_name="fixtures_tag_set")
510+    tagged_id = models.PositiveIntegerField(default=0)
511+    tagged = generic.GenericForeignKey(ct_field='tagged_type',
512+                                       fk_field='tagged_id')
513+
514+    def __unicode__(self):
515+        return '<%s: %s> tagged "%s"' % (self.tagged.__class__.__name__,
516+                                         self.tagged, self.name)
517+
518+class PersonManager(models.Manager):
519+    def get_by_surrogate(self, name):
520+        return self.get(name=name)
521+
522+class Person(models.Model):
523+    objects = PersonManager()
524+    name = models.CharField(max_length=100)
525+    def __unicode__(self):
526+        return self.name
527+
528+    class Meta:
529+        ordering = ('name',)
530+
531+    def surrogate_key(self):
532+        return self.name
533+
534+class Visa(models.Model):
535+    person = models.ForeignKey(Person)
536+    permissions = models.ManyToManyField(Permission, blank=True)
537+
538+    def __unicode__(self):
539+        return '%s %s' % (self.person.name,
540+                          ', '.join(p.name for p in self.permissions.all()))
541+
542+class Book(models.Model):
543+    name = models.CharField(max_length=100)
544+    authors = models.ManyToManyField(Person)
545+
546+    def __unicode__(self):
547+        return '%s by %s' % (self.name,
548+                          ' and '.join(a.name for a in self.authors.all()))
549+
550+    class Meta:
551+        ordering = ('name',)
552+
553 __test__ = {'API_TESTS': """
554 >>> from django.core import management
555 >>> from django.db.models import get_app
556@@ -90,12 +149,45 @@
557 >>> Article.objects.all()
558 [<Article: XML identified as leading cause of cancer>, <Article: Django conquers world!>, <Article: Copyright is fine the way it is>, <Article: Poker on TV is great!>, <Article: Python program becomes self aware>]
559 
560+# Load fixture 6, JSON file with dynamic ContentType fields. Testing ManyToOne.
561+>>> management.call_command('loaddata', 'fixture6.json', verbosity=0)
562+>>> Tag.objects.all()
563+[<Tag: <Article: Copyright is fine the way it is> tagged "copyright">, <Tag: <Article: Copyright is fine the way it is> tagged "law">]
564+
565+# Load fixture 7, XML file with dynamic ContentType fields. Testing ManyToOne.
566+>>> management.call_command('loaddata', 'fixture7.xml', verbosity=0)
567+>>> Tag.objects.all()
568+[<Tag: <Article: Copyright is fine the way it is> tagged "copyright">, <Tag: <Article: Copyright is fine the way it is> tagged "legal">, <Tag: <Article: Django conquers world!> tagged "django">, <Tag: <Article: Django conquers world!> tagged "world domination">]
569+
570+# Load fixture 8, JSON file with dynamic Permission fields. Testing ManyToMany.
571+>>> management.call_command('loaddata', 'fixture8.json', verbosity=0)
572+>>> Visa.objects.all()
573+[<Visa: Django Reinhardt Can add user, Can change user, Can delete user>, <Visa: Stephane Grappelli Can add user>, <Visa: Prince >]
574+
575+# Load fixture 9, XML file with dynamic Permission fields. Testing ManyToMany.
576+>>> management.call_command('loaddata', 'fixture9.xml', verbosity=0)
577+>>> Visa.objects.all()
578+[<Visa: Django Reinhardt Can add user, Can change user, Can delete user>, <Visa: Stephane Grappelli Can add user, Can delete user>, <Visa: Artist formerly known as "Prince" Can change user>]
579+
580+>>> Book.objects.all()
581+[<Book: Music for all ages by Artist formerly known as "Prince" and Django Reinhardt>]
582+
583 # Load a fixture that doesn't exist
584 >>> management.call_command('loaddata', 'unknown.json', verbosity=0)
585 
586 # object list is unaffected
587 >>> Article.objects.all()
588 [<Article: XML identified as leading cause of cancer>, <Article: Django conquers world!>, <Article: Copyright is fine the way it is>, <Article: Poker on TV is great!>, <Article: Python program becomes self aware>]
589+
590+# Dump the current contents of the database as a JSON fixture
591+>>> management.call_command('dumpdata', 'fixtures', format='json')
592+[{"pk": 3, "model": "fixtures.person", "fields": {"name": "Artist formerly known as \\"Prince\\""}}, {"pk": 1, "model": "fixtures.person", "fields": {"name": "Django Reinhardt"}}, {"pk": 2, "model": "fixtures.person", "fields": {"name": "Stephane Grappelli"}}, {"pk": 1, "model": "fixtures.category", "fields": {"description": "Latest news stories", "title": "News Stories"}}, {"pk": 5, "model": "fixtures.article", "fields": {"headline": "XML identified as leading cause of cancer", "pub_date": "2006-06-16 16:00:00"}}, {"pk": 4, "model": "fixtures.article", "fields": {"headline": "Django conquers world!", "pub_date": "2006-06-16 15:00:00"}}, {"pk": 3, "model": "fixtures.article", "fields": {"headline": "Copyright is fine the way it is", "pub_date": "2006-06-16 14:00:00"}}, {"pk": 2, "model": "fixtures.article", "fields": {"headline": "Poker on TV is great!", "pub_date": "2006-06-16 11:00:00"}}, {"pk": 1, "model": "fixtures.article", "fields": {"headline": "Python program becomes self aware", "pub_date": "2006-06-16 11:00:00"}}, {"pk": 1, "model": "fixtures.tag", "fields": {"tagged_type": ["fixtures", "article"], "name": "copyright", "tagged_id": 3}}, {"pk": 2, "model": "fixtures.tag", "fields": {"tagged_type": ["fixtures", "article"], "name": "legal", "tagged_id": 3}}, {"pk": 3, "model": "fixtures.tag", "fields": {"tagged_type": ["fixtures", "article"], "name": "django", "tagged_id": 4}}, {"pk": 4, "model": "fixtures.tag", "fields": {"tagged_type": ["fixtures", "article"], "name": "world domination", "tagged_id": 4}}, {"pk": 1, "model": "fixtures.visa", "fields": {"person": "Django Reinhardt", "permissions": [["add_user", "auth", "user"], ["change_user", "auth", "user"], ["delete_user", "auth", "user"]]}}, {"pk": 2, "model": "fixtures.visa", "fields": {"person": "Stephane Grappelli", "permissions": [["add_user", "auth", "user"], ["delete_user", "auth", "user"]]}}, {"pk": 3, "model": "fixtures.visa", "fields": {"person": "Artist formerly known as \\"Prince\\"", "permissions": [["change_user", "auth", "user"]]}}, {"pk": 1, "model": "fixtures.book", "fields": {"name": "Music for all ages", "authors": ["Artist formerly known as \\"Prince\\"", "Django Reinhardt"]}}]
593+
594+# Dump the current contents of the database as an XML fixture
595+>>> management.call_command('dumpdata', 'fixtures', format='xml')
596+<?xml version="1.0" encoding="utf-8"?>
597+<django-objects version="1.0"><object pk="3" model="fixtures.person"><field type="CharField" name="name">Artist formerly known as "Prince"</field></object><object pk="1" model="fixtures.person"><field type="CharField" name="name">Django Reinhardt</field></object><object pk="2" model="fixtures.person"><field type="CharField" name="name">Stephane Grappelli</field></object><object pk="1" model="fixtures.category"><field type="CharField" name="title">News Stories</field><field type="TextField" name="description">Latest news stories</field></object><object pk="5" model="fixtures.article"><field type="CharField" name="headline">XML identified as leading cause of cancer</field><field type="DateTimeField" name="pub_date">2006-06-16 16:00:00</field></object><object pk="4" model="fixtures.article"><field type="CharField" name="headline">Django conquers world!</field><field type="DateTimeField" name="pub_date">2006-06-16 15:00:00</field></object><object pk="3" model="fixtures.article"><field type="CharField" name="headline">Copyright is fine the way it is</field><field type="DateTimeField" name="pub_date">2006-06-16 14:00:00</field></object><object pk="2" model="fixtures.article"><field type="CharField" name="headline">Poker on TV is great!</field><field type="DateTimeField" name="pub_date">2006-06-16 11:00:00</field></object><object pk="1" model="fixtures.article"><field type="CharField" name="headline">Python program becomes self aware</field><field type="DateTimeField" name="pub_date">2006-06-16 11:00:00</field></object><object pk="1" model="fixtures.tag"><field type="CharField" name="name">copyright</field><field to="contenttypes.contenttype" name="tagged_type" rel="ManyToOneRel"><surrogate value="fixtures"></surrogate><surrogate value="article"></surrogate></field><field type="PositiveIntegerField" name="tagged_id">3</field></object><object pk="2" model="fixtures.tag"><field type="CharField" name="name">legal</field><field to="contenttypes.contenttype" name="tagged_type" rel="ManyToOneRel"><surrogate value="fixtures"></surrogate><surrogate value="article"></surrogate></field><field type="PositiveIntegerField" name="tagged_id">3</field></object><object pk="3" model="fixtures.tag"><field type="CharField" name="name">django</field><field to="contenttypes.contenttype" name="tagged_type" rel="ManyToOneRel"><surrogate value="fixtures"></surrogate><surrogate value="article"></surrogate></field><field type="PositiveIntegerField" name="tagged_id">4</field></object><object pk="4" model="fixtures.tag"><field type="CharField" name="name">world domination</field><field to="contenttypes.contenttype" name="tagged_type" rel="ManyToOneRel"><surrogate value="fixtures"></surrogate><surrogate value="article"></surrogate></field><field type="PositiveIntegerField" name="tagged_id">4</field></object><object pk="1" model="fixtures.visa"><field to="fixtures.person" name="person" rel="ManyToOneRel">Django Reinhardt</field><field to="auth.permission" name="permissions" rel="ManyToManyRel"><object><surrogate value="add_user"></surrogate><surrogate value="auth"></surrogate><surrogate value="user"></surrogate></object><object><surrogate value="change_user"></surrogate><surrogate value="auth"></surrogate><surrogate value="user"></surrogate></object><object><surrogate value="delete_user"></surrogate><surrogate value="auth"></surrogate><surrogate value="user"></surrogate></object></field></object><object pk="2" model="fixtures.visa"><field to="fixtures.person" name="person" rel="ManyToOneRel">Stephane Grappelli</field><field to="auth.permission" name="permissions" rel="ManyToManyRel"><object><surrogate value="add_user"></surrogate><surrogate value="auth"></surrogate><surrogate value="user"></surrogate></object><object><surrogate value="delete_user"></surrogate><surrogate value="auth"></surrogate><surrogate value="user"></surrogate></object></field></object><object pk="3" model="fixtures.visa"><field to="fixtures.person" name="person" rel="ManyToOneRel">Artist formerly known as "Prince"</field><field to="auth.permission" name="permissions" rel="ManyToManyRel"><object><surrogate value="change_user"></surrogate><surrogate value="auth"></surrogate><surrogate value="user"></surrogate></object></field></object><object pk="1" model="fixtures.book"><field type="CharField" name="name">Music for all ages</field><field to="fixtures.person" name="authors" rel="ManyToManyRel"><object surrogate='Artist formerly known as "Prince"'></object><object surrogate="Django Reinhardt"></object></field></object></django-objects>
598+
599 """}
600 
601 # Database flushing does not work on MySQL with the default storage engine
602@@ -159,6 +251,25 @@
603 >>> management.call_command('loaddata', 'fixture5', verbosity=0) # doctest: +ELLIPSIS
604 Multiple fixtures named 'fixture5' in '...fixtures'. Aborting.
605 
606+>>> management.call_command('flush', verbosity=0, interactive=False)
607+
608+# Load back in fixture 1, we need the articles from it
609+>>> management.call_command('loaddata', 'fixture1', verbosity=0)
610+
611+# Try to load fixture 6 using format discovery
612+>>> management.call_command('loaddata', 'fixture6', verbosity=0)
613+>>> Tag.objects.all()
614+[<Tag: <Article: Time to reform copyright> tagged "copyright">, <Tag: <Article: Time to reform copyright> tagged "law">]
615+
616+# Dump the current contents of the database as a JSON fixture
617+>>> management.call_command('dumpdata', 'fixtures', format='json')
618+[{"pk": 1, "model": "fixtures.person", "fields": {"name": "Django Reinhardt"}}, {"pk": 3, "model": "fixtures.person", "fields": {"name": "Prince"}}, {"pk": 2, "model": "fixtures.person", "fields": {"name": "Stephane Grappelli"}}, {"pk": 1, "model": "fixtures.category", "fields": {"description": "Latest news stories", "title": "News Stories"}}, {"pk": 3, "model": "fixtures.article", "fields": {"headline": "Time to reform copyright", "pub_date": "2006-06-16 13:00:00"}}, {"pk": 2, "model": "fixtures.article", "fields": {"headline": "Poker has no place on ESPN", "pub_date": "2006-06-16 12:00:00"}}, {"pk": 1, "model": "fixtures.article", "fields": {"headline": "Python program becomes self aware", "pub_date": "2006-06-16 11:00:00"}}, {"pk": 1, "model": "fixtures.tag", "fields": {"tagged_type": ["fixtures", "article"], "name": "copyright", "tagged_id": 3}}, {"pk": 2, "model": "fixtures.tag", "fields": {"tagged_type": ["fixtures", "article"], "name": "law", "tagged_id": 3}}]
619+
620+# Dump the current contents of the database as an XML fixture
621+>>> management.call_command('dumpdata', 'fixtures', format='xml')
622+<?xml version="1.0" encoding="utf-8"?>
623+<django-objects version="1.0"><object pk="1" model="fixtures.person"><field type="CharField" name="name">Django Reinhardt</field></object><object pk="3" model="fixtures.person"><field type="CharField" name="name">Prince</field></object><object pk="2" model="fixtures.person"><field type="CharField" name="name">Stephane Grappelli</field></object><object pk="1" model="fixtures.category"><field type="CharField" name="title">News Stories</field><field type="TextField" name="description">Latest news stories</field></object><object pk="3" model="fixtures.article"><field type="CharField" name="headline">Time to reform copyright</field><field type="DateTimeField" name="pub_date">2006-06-16 13:00:00</field></object><object pk="2" model="fixtures.article"><field type="CharField" name="headline">Poker has no place on ESPN</field><field type="DateTimeField" name="pub_date">2006-06-16 12:00:00</field></object><object pk="1" model="fixtures.article"><field type="CharField" name="headline">Python program becomes self aware</field><field type="DateTimeField" name="pub_date">2006-06-16 11:00:00</field></object><object pk="1" model="fixtures.tag"><field type="CharField" name="name">copyright</field><field to="contenttypes.contenttype" name="tagged_type" rel="ManyToOneRel"><surrogate value="fixtures"></surrogate><surrogate value="article"></surrogate></field><field type="PositiveIntegerField" name="tagged_id">3</field></object><object pk="2" model="fixtures.tag"><field type="CharField" name="name">law</field><field to="contenttypes.contenttype" name="tagged_type" rel="ManyToOneRel"><surrogate value="fixtures"></surrogate><surrogate value="article"></surrogate></field><field type="PositiveIntegerField" name="tagged_id">3</field></object></django-objects>
624+
625 """
626 
627 from django.test import TestCase
628diff -r f266dfb70b06 tests/regressiontests/fixtures_regress/fixtures/forward_ref_lookup.json
629--- /dev/null   Thu Jan 01 00:00:00 1970 +0000
630+++ b/tests/regressiontests/fixtures_regress/fixtures/forward_ref_lookup.json   Sat Dec 05 01:25:01 2009 +0800
631@@ -0,0 +1,32 @@
632+[
633+    {
634+        "pk": "4",
635+        "model": "fixtures_regress.person",
636+        "fields": {
637+            "name": "Neal Stephenson"
638+        }
639+    },
640+    {
641+        "pk": "2",
642+        "model": "fixtures_regress.store",
643+        "fields": {
644+            "name": "Amazon"
645+        }
646+    },
647+    {
648+        "pk": "3",
649+        "model": "fixtures_regress.store",
650+        "fields": {
651+            "name": "Borders"
652+        }
653+    },
654+    {
655+        "pk": 1,
656+        "model": "fixtures_regress.book",
657+        "fields": {
658+            "name": "Cryptonomicon",
659+            "author": "Neal Stephenson",
660+            "stores": ["Amazon", "Borders"]
661+        }
662+    }
663+]
664\ No newline at end of file
665diff -r f266dfb70b06 tests/regressiontests/fixtures_regress/models.py
666--- a/tests/regressiontests/fixtures_regress/models.py  Wed Nov 04 11:38:06 2009 +0000
667+++ b/tests/regressiontests/fixtures_regress/models.py  Sat Dec 05 01:25:01 2009 +0800
668@@ -13,7 +13,7 @@
669     specimens = models.Manager()
670 
671     def __unicode__(self):
672-        return self.common_name
673+        return self.name
674 
675 def animal_pre_save_check(signal, sender, instance, **kwargs):
676     "A signal that is used to check the type of data loaded from fixtures"
677@@ -69,10 +69,60 @@
678 class Widget(models.Model):
679     name = models.CharField(max_length=255)
680 
681+    class Meta:
682+        ordering = ('name',)
683+
684+    def __unicode__(self):
685+        return self.name
686+
687 class WidgetProxy(Widget):
688     class Meta:
689         proxy = True
690 
691+# Check for forward references in FKs and M2Ms with surrogate keys
692+
693+class TestManager(models.Manager):
694+    def get_by_surrogate(self, key):
695+        return self.get(name=key)
696+
697+class Store(models.Model):
698+    objects = TestManager()
699+    name = models.CharField(max_length=255)
700+
701+    class Meta:
702+        ordering = ('name',)
703+
704+    def __unicode__(self):
705+        return self.name
706+
707+    def surrogate_key(self):
708+        return self.name
709+
710+class Person(models.Model):
711+    objects = TestManager()
712+    name = models.CharField(max_length=255)
713+
714+    class Meta:
715+        ordering = ('name',)
716+
717+    def __unicode__(self):
718+        return self.name
719+
720+    # Person doesn't actually have a dependency on store, but we need to define
721+    # one to test the behaviour of the dependency resolution algorithm.
722+    def surrogate_key(self):
723+        return self.name
724+    surrogate_key.dependencies = ['fixtures_regress.store']
725+
726+class Book(models.Model):
727+    name = models.CharField(max_length=255)
728+    author = models.ForeignKey(Person)
729+    stores = models.ManyToManyField(Store)
730+
731+    class Meta:
732+        ordering = ('name',)
733+
734+
735 __test__ = {'API_TESTS':"""
736 >>> from django.core import management
737 
738@@ -192,4 +242,92 @@
739 >>> management.call_command('dumpdata', 'fixtures_regress', format='json')
740 [{"pk": 1, "model": "fixtures_regress.widget", "fields": {"name": "grommet"}}]
741 
742+###############################################
743+# Check that surrogate requirements are taken into account
744+# when serializing models
745+>>> management.call_command('loaddata', 'forward_ref_lookup.json', verbosity=0)
746+
747+>>> management.call_command('dumpdata', 'fixtures_regress.book', 'fixtures_regress.person', 'fixtures_regress.store', verbosity=0)
748+[{"pk": 2, "model": "fixtures_regress.store", "fields": {"name": "Amazon"}}, {"pk": 3, "model": "fixtures_regress.store", "fields": {"name": "Borders"}}, {"pk": 4, "model": "fixtures_regress.person", "fields": {"name": "Neal Stephenson"}}, {"pk": 1, "model": "fixtures_regress.book", "fields": {"stores": ["Amazon", "Borders"], "name": "Cryptonomicon", "author": "Neal Stephenson"}}]
749+
750+# Now lets check the dependency sorting explicitly
751+
752+# First Some models with pathological circular dependencies
753+>>> class Circle1(models.Model):
754+...     name = models.CharField(max_length=255)
755+...     def surrogate_key(self):
756+...         return self.name
757+...     surrogate_key.dependencies = ['fixtures_regress.circle2']
758+
759+>>> class Circle2(models.Model):
760+...     name = models.CharField(max_length=255)
761+...     def surrogate_key(self):
762+...         return self.name
763+...     surrogate_key.dependencies = ['fixtures_regress.circle1']
764+
765+>>> class Circle3(models.Model):
766+...     name = models.CharField(max_length=255)
767+...     def surrogate_key(self):
768+...         return self.name
769+...     surrogate_key.dependencies = ['fixtures_regress.circle3']
770+
771+>>> class Circle4(models.Model):
772+...     name = models.CharField(max_length=255)
773+...     def surrogate_key(self):
774+...         return self.name
775+...     surrogate_key.dependencies = ['fixtures_regress.circle5']
776+
777+>>> class Circle5(models.Model):
778+...     name = models.CharField(max_length=255)
779+...     def surrogate_key(self):
780+...         return self.name
781+...     surrogate_key.dependencies = ['fixtures_regress.circle6']
782+
783+>>> class Circle6(models.Model):
784+...     name = models.CharField(max_length=255)
785+...     def surrogate_key(self):
786+...         return self.name
787+...     surrogate_key.dependencies = ['fixtures_regress.circle4']
788+
789+# It doesn't matter what order you mention the models
790+# Store *must* be serialized first, then person, then book.
791+>>> from django.core.management.commands.dumpdata import sort_dependencies
792+>>> sort_dependencies([('fixtures_regress', [Book, Person, Store])])
793+[<class 'regressiontests.fixtures_regress.models.Store'>, <class 'regressiontests.fixtures_regress.models.Person'>, <class 'regressiontests.fixtures_regress.models.Book'>]
794+
795+>>> sort_dependencies([('fixtures_regress', [Store, Book, Person])])
796+[<class 'regressiontests.fixtures_regress.models.Store'>, <class 'regressiontests.fixtures_regress.models.Person'>, <class 'regressiontests.fixtures_regress.models.Book'>]
797+
798+>>> sort_dependencies([('fixtures_regress', [Person, Store, Book])])
799+[<class 'regressiontests.fixtures_regress.models.Store'>, <class 'regressiontests.fixtures_regress.models.Person'>, <class 'regressiontests.fixtures_regress.models.Book'>]
800+
801+# A dangling dependency
802+>>> sort_dependencies([('fixtures_regress', [Person, Circle1, Store, Book])])
803+Traceback (most recent call last):
804+...
805+CommandError: Can't resolve dependencies for fixtures_regress.Circle1 in serialized app list.
806+
807+# A tight circular dependency
808+>>> sort_dependencies([('fixtures_regress', [Person, Circle2, Circle1, Store, Book])])
809+Traceback (most recent call last):
810+...
811+CommandError: Can't resolve dependencies for fixtures_regress.Circle2, fixtures_regress.Circle1 in serialized app list.
812+
813+>>> sort_dependencies([('fixtures_regress', [Circle1, Book, Circle2])])
814+Traceback (most recent call last):
815+...
816+CommandError: Can't resolve dependencies for fixtures_regress.Circle2, fixtures_regress.Circle1 in serialized app list.
817+
818+# A self referential dependency
819+>>> sort_dependencies([('fixtures_regress', [Book, Circle3])])
820+Traceback (most recent call last):
821+...
822+CommandError: Can't resolve dependencies for fixtures_regress.Circle3 in serialized app list.
823+
824+# A long circular dependency
825+>>> sort_dependencies([('fixtures_regress', [Person, Circle2, Circle1, Circle3, Store, Book])])
826+Traceback (most recent call last):
827+...
828+CommandError: Can't resolve dependencies for fixtures_regress.Circle2, fixtures_regress.Circle1, fixtures_regress.Circle3 in serialized app list.
829+
830 """}