Code

Ticket #3163: create_db_schema-trunk7027-withtests.2.diff

File create_db_schema-trunk7027-withtests.2.diff, 24.7 KB (added by honeyman, 6 years ago)

create_db_schema Mera option implementation (svn diff, Django trunk 7027, tests included)

Line 
1Index: django/db/models/options.py
2===================================================================
3--- django/db/models/options.py (revision 7027)
4+++ django/db/models/options.py (working copy)
5@@ -15,7 +15,7 @@
6 
7 DEFAULT_NAMES = ('verbose_name', 'db_table', 'ordering',
8                  'unique_together', 'permissions', 'get_latest_by',
9-                 'order_with_respect_to', 'app_label', 'db_tablespace')
10+                 'order_with_respect_to', 'app_label', 'db_tablespace', 'create_db_schema')
11 
12 class Options(object):
13     def __init__(self, meta):
14@@ -30,6 +30,7 @@
15         self.get_latest_by = None
16         self.order_with_respect_to = None
17         self.db_tablespace = settings.DEFAULT_TABLESPACE
18+        self.create_db_schema = True
19         self.admin = None
20         self.meta = meta
21         self.pk = None
22Index: django/core/management/commands/syncdb.py
23===================================================================
24--- django/core/management/commands/syncdb.py   (revision 7027)
25+++ django/core/management/commands/syncdb.py   (working copy)
26@@ -57,22 +57,23 @@
27             app_name = app.__name__.split('.')[-2]
28             model_list = models.get_models(app)
29             for model in model_list:
30-                # Create the model's database table, if it doesn't already exist.
31-                if verbosity >= 2:
32-                    print "Processing %s.%s model" % (app_name, model._meta.object_name)
33-                if table_name_converter(model._meta.db_table) in tables:
34-                    continue
35-                sql, references = sql_model_create(model, self.style, seen_models)
36-                seen_models.add(model)
37-                created_models.add(model)
38-                for refto, refs in references.items():
39-                    pending_references.setdefault(refto, []).extend(refs)
40-                sql.extend(sql_for_pending_references(model, self.style, pending_references))
41-                if verbosity >= 1:
42-                    print "Creating table %s" % model._meta.db_table
43-                for statement in sql:
44-                    cursor.execute(statement)
45-                tables.append(table_name_converter(model._meta.db_table))
46+                if model._meta.create_db_schema:
47+                    # Create the model's database table, if it doesn't already exist.
48+                    if verbosity >= 2:
49+                        print "Processing %s.%s model" % (app_name, model._meta.object_name)
50+                    if table_name_converter(model._meta.db_table) in tables:
51+                        continue
52+                    sql, references = sql_model_create(model, self.style, seen_models)
53+                    seen_models.add(model)
54+                    created_models.add(model)
55+                    for refto, refs in references.items():
56+                        pending_references.setdefault(refto, []).extend(refs)
57+                    sql.extend(sql_for_pending_references(model, self.style, pending_references))
58+                    if verbosity >= 1:
59+                        print "Creating table %s" % model._meta.db_table
60+                    for statement in sql:
61+                        cursor.execute(statement)
62+                    tables.append(table_name_converter(model._meta.db_table))
63 
64         # Create the m2m tables. This must be done after all tables have been created
65         # to ensure that all referred tables will exist.
66@@ -119,19 +120,20 @@
67             app_name = app.__name__.split('.')[-2]
68             for model in models.get_models(app):
69                 if model in created_models:
70-                    index_sql = sql_indexes_for_model(model, self.style)
71-                    if index_sql:
72-                        if verbosity >= 1:
73-                            print "Installing index for %s.%s model" % (app_name, model._meta.object_name)
74-                        try:
75-                            for sql in index_sql:
76-                                cursor.execute(sql)
77-                        except Exception, e:
78-                            sys.stderr.write("Failed to install index for %s.%s model: %s" % \
79-                                                (app_name, model._meta.object_name, e))
80-                            transaction.rollback_unless_managed()
81-                        else:
82-                            transaction.commit_unless_managed()
83+                    if model._meta.create_db_schema:
84+                        index_sql = sql_indexes_for_model(model, self.style)
85+                        if index_sql:
86+                            if verbosity >= 1:
87+                                print "Installing index for %s.%s model" % (app_name, model._meta.object_name)
88+                            try:
89+                                for sql in index_sql:
90+                                    cursor.execute(sql)
91+                            except Exception, e:
92+                                sys.stderr.write("Failed to install index for %s.%s model: %s" % \
93+                                                    (app_name, model._meta.object_name, e))
94+                                transaction.rollback_unless_managed()
95+                            else:
96+                                transaction.commit_unless_managed()
97 
98         # Install the 'initial_data' fixture, using format discovery
99         from django.core.management import call_command
100Index: django/core/management/sql.py
101===================================================================
102--- django/core/management/sql.py       (revision 7027)
103+++ django/core/management/sql.py       (working copy)
104@@ -13,20 +13,24 @@
105     cursor = connection.cursor()
106     return get_introspection_module().get_table_list(cursor)
107 
108-def django_table_list(only_existing=False):
109+def django_table_list(only_existing=False, filter_not_generated_tables=False):
110     """
111     Returns a list of all table names that have associated Django models and
112     are in INSTALLED_APPS.
113 
114     If only_existing is True, the resulting list will only include the tables
115     that actually exist in the database.
116+
117+    If filter_not_generated_tables is True, then all tables with associated Django models
118+    which have Meta option create_db_schema=False will not be added to the list.
119     """
120     from django.db import models
121     tables = []
122     for app in models.get_apps():
123         for model in models.get_models(app):
124-            tables.append(model._meta.db_table)
125-            tables.extend([f.m2m_db_table() for f in model._meta.many_to_many])
126+            if (not filter_not_generated_tables) or (model._meta.create_db_schema):
127+                tables.append(model._meta.db_table)
128+                tables.extend([f.m2m_db_table() for f in model._meta.many_to_many])
129     if only_existing:
130         existing = table_list()
131         tables = [t for t in tables if t in existing]
132@@ -45,8 +49,13 @@
133         converter = lambda x: x
134     return set([m for m in all_models if converter(m._meta.db_table) in map(converter, table_list)])
135 
136-def sequence_list():
137-    "Returns a list of information about all DB sequences for all models in all apps."
138+def sequence_list(filter_not_generated_tables=False):
139+    """
140+    Returns a list of information about all DB sequences for all models in all apps.
141+
142+    If filter_not_generated_tables is True, then only the sequences for the Django models
143+    which have Meta option create_db_schema=False will be added to the list.
144+    """
145     from django.db import models
146 
147     apps = models.get_apps()
148@@ -54,10 +63,11 @@
149 
150     for app in apps:
151         for model in models.get_models(app):
152-            for f in model._meta.fields:
153-                if isinstance(f, models.AutoField):
154-                    sequence_list.append({'table': model._meta.db_table, 'column': f.column})
155-                    break # Only one AutoField is allowed per model, so don't bother continuing.
156+            if (not filter_not_generated_tables) or (model._meta.create_db_schema):
157+                for f in model._meta.fields:
158+                    if isinstance(f, models.AutoField):
159+                        sequence_list.append({'table': model._meta.db_table, 'column': f.column})
160+                        break # Only one AutoField is allowed per model, so don't bother continuing.
161 
162             for f in model._meta.many_to_many:
163                 sequence_list.append({'table': f.m2m_db_table(), 'column': None})
164@@ -156,8 +166,9 @@
165     for model in app_models:
166         if cursor and table_name_converter(model._meta.db_table) in table_names:
167             # Drop the table now
168-            output.append('%s %s;' % (style.SQL_KEYWORD('DROP TABLE'),
169-                style.SQL_TABLE(qn(model._meta.db_table))))
170+            if model._meta.create_db_schema:
171+                output.append('%s %s;' % (style.SQL_KEYWORD('DROP TABLE'),
172+                    style.SQL_TABLE(qn(model._meta.db_table))))
173             if connection.features.supports_constraints and model in references_to_delete:
174                 for rel_class, f in references_to_delete[model]:
175                     table = rel_class._meta.db_table
176@@ -165,11 +176,12 @@
177                     r_table = model._meta.db_table
178                     r_col = model._meta.get_field(f.rel.field_name).column
179                     r_name = '%s_refs_%s_%x' % (col, r_col, abs(hash((table, r_table))))
180-                    output.append('%s %s %s %s;' % \
181-                        (style.SQL_KEYWORD('ALTER TABLE'),
182-                        style.SQL_TABLE(qn(table)),
183-                        style.SQL_KEYWORD(connection.ops.drop_foreignkey_sql()),
184-                        style.SQL_FIELD(truncate_name(r_name, connection.ops.max_name_length()))))
185+                    if rel_class._meta.create_db_schema:
186+                        output.append('%s %s %s %s;' % \
187+                            (style.SQL_KEYWORD('ALTER TABLE'),
188+                            style.SQL_TABLE(qn(table)),
189+                            style.SQL_KEYWORD(connection.ops.drop_foreignkey_sql()),
190+                            style.SQL_FIELD(truncate_name(r_name, connection.ops.max_name_length()))))
191                 del references_to_delete[model]
192             if model._meta.has_auto_field:
193                 ds = connection.ops.drop_sequence_sql(model._meta.db_table)
194@@ -206,16 +218,16 @@
195 def sql_flush(style, only_django=False):
196     """
197     Returns a list of the SQL statements used to flush the database.
198-   
199+
200     If only_django is True, then only table names that have associated Django
201     models and are in INSTALLED_APPS will be included.
202     """
203     from django.db import connection
204     if only_django:
205-        tables = django_table_list()
206+        tables = django_table_list(filter_not_generated_tables=True)
207     else:
208         tables = table_list()
209-    statements = connection.ops.sql_flush(style, tables, sequence_list())
210+    statements = connection.ops.sql_flush(style, tables, sequence_list(filter_not_generated_tables=True))
211     return statements
212 
213 def sql_custom(app):
214@@ -295,22 +307,24 @@
215         table_output.append(style.SQL_KEYWORD('UNIQUE') + ' (%s)' % \
216             ", ".join([style.SQL_FIELD(qn(opts.get_field(f).column)) for f in field_constraints]))
217 
218-    full_statement = [style.SQL_KEYWORD('CREATE TABLE') + ' ' + style.SQL_TABLE(qn(opts.db_table)) + ' (']
219-    for i, line in enumerate(table_output): # Combine and add commas.
220-        full_statement.append('    %s%s' % (line, i < len(table_output)-1 and ',' or ''))
221-    full_statement.append(')')
222-    if opts.db_tablespace and connection.features.supports_tablespaces:
223-        full_statement.append(connection.ops.tablespace_sql(opts.db_tablespace))
224-    full_statement.append(';')
225-    final_output.append('\n'.join(full_statement))
226+    # Now build up the CREATE TABLE section but only if the model requires it
227+    if opts.create_db_schema:
228+        full_statement = [style.SQL_KEYWORD('CREATE TABLE') + ' ' + style.SQL_TABLE(qn(opts.db_table)) + ' (']
229+        for i, line in enumerate(table_output): # Combine and add commas.
230+            full_statement.append('    %s%s' % (line, i < len(table_output)-1 and ',' or ''))
231+        full_statement.append(')')
232+        if opts.db_tablespace and connection.features.supports_tablespaces:
233+            full_statement.append(connection.ops.tablespace_sql(opts.db_tablespace))
234+        full_statement.append(';')
235+        final_output.append('\n'.join(full_statement))
236 
237-    if opts.has_auto_field:
238-        # Add any extra SQL needed to support auto-incrementing primary keys.
239-        auto_column = opts.auto_field.db_column or opts.auto_field.name
240-        autoinc_sql = connection.ops.autoinc_sql(opts.db_table, auto_column)
241-        if autoinc_sql:
242-            for stmt in autoinc_sql:
243-                final_output.append(stmt)
244+        if opts.has_auto_field:
245+            # Add any extra SQL needed to support auto-incrementing primary keys.
246+            auto_column = opts.auto_field.db_column or opts.auto_field.name
247+            autoinc_sql = connection.ops.autoinc_sql(opts.db_table, auto_column)
248+            if autoinc_sql:
249+                for stmt in autoinc_sql:
250+                    final_output.append(stmt)
251 
252     return final_output, pending_references
253 
254@@ -335,10 +349,11 @@
255                 # For MySQL, r_name must be unique in the first 64 characters.
256                 # So we are careful with character usage here.
257                 r_name = '%s_refs_%s_%x' % (r_col, col, abs(hash((r_table, table))))
258-                final_output.append(style.SQL_KEYWORD('ALTER TABLE') + ' %s ADD CONSTRAINT %s FOREIGN KEY (%s) REFERENCES %s (%s)%s;' % \
259-                    (qn(r_table), truncate_name(r_name, connection.ops.max_name_length()),
260-                    qn(r_col), qn(table), qn(col),
261-                    connection.ops.deferrable_sql()))
262+                if rel_opts.create_db_schema:
263+                    final_output.append(style.SQL_KEYWORD('ALTER TABLE') + ' %s ADD CONSTRAINT %s FOREIGN KEY (%s) REFERENCES %s (%s)%s;' % \
264+                        (qn(r_table), truncate_name(r_name, connection.ops.max_name_length()),
265+                        qn(r_col), qn(table), qn(col),
266+                        connection.ops.deferrable_sql()))
267             del pending_references[model]
268     return final_output
269 
270@@ -411,7 +426,7 @@
271             for r_table, r_col, table, col in deferred:
272                 r_name = '%s_refs_%s_%x' % (r_col, col,
273                         abs(hash((r_table, table))))
274-                final_output.append(style.SQL_KEYWORD('ALTER TABLE') + ' %s ADD CONSTRAINT %s FOREIGN KEY (%s) REFERENCES %s (%s)%s;' %
275+                final_output.append(style.SQL_KEYWORD('ALTER TABLE') + ' %s ADD CONSTRAINT %s FOREIGN KEY (%s) REFERENCES %s (%s)%s;' %
276                 (qn(r_table),
277                 truncate_name(r_name, connection.ops.max_name_length()),
278                 qn(r_col), qn(table), qn(col),
279@@ -458,22 +473,23 @@
280     output = []
281 
282     qn = connection.ops.quote_name
283-    for f in model._meta.fields:
284-        if f.db_index and not ((f.primary_key or f.unique) and connection.features.autoindexes_primary_keys):
285-            unique = f.unique and 'UNIQUE ' or ''
286-            tablespace = f.db_tablespace or model._meta.db_tablespace
287-            if tablespace and connection.features.supports_tablespaces:
288-                tablespace_sql = ' ' + connection.ops.tablespace_sql(tablespace)
289-            else:
290-                tablespace_sql = ''
291-            output.append(
292-                style.SQL_KEYWORD('CREATE %sINDEX' % unique) + ' ' + \
293-                style.SQL_TABLE(qn('%s_%s' % (model._meta.db_table, f.column))) + ' ' + \
294-                style.SQL_KEYWORD('ON') + ' ' + \
295-                style.SQL_TABLE(qn(model._meta.db_table)) + ' ' + \
296-                "(%s)" % style.SQL_FIELD(qn(f.column)) + \
297-                "%s;" % tablespace_sql
298-            )
299+    if model._meta.create_db_schema:
300+        for f in model._meta.fields:
301+            if f.db_index and not ((f.primary_key or f.unique) and connection.features.autoindexes_primary_keys):
302+                unique = f.unique and 'UNIQUE ' or ''
303+                tablespace = f.db_tablespace or model._meta.db_tablespace
304+                if tablespace and connection.features.supports_tablespaces:
305+                    tablespace_sql = ' ' + connection.ops.tablespace_sql(tablespace)
306+                else:
307+                    tablespace_sql = ''
308+                output.append(
309+                    style.SQL_KEYWORD('CREATE %sINDEX' % unique) + ' ' + \
310+                    style.SQL_TABLE(qn('%s_%s' % (model._meta.db_table, f.column))) + ' ' + \
311+                    style.SQL_KEYWORD('ON') + ' ' + \
312+                    style.SQL_TABLE(qn(model._meta.db_table)) + ' ' + \
313+                    "(%s)" % style.SQL_FIELD(qn(f.column)) + \
314+                    "%s;" % tablespace_sql
315+                )
316     return output
317 
318 def emit_post_sync_signal(created_models, verbosity, interactive):
319Index: tests/modeltests/create_db_schema/__init__.py
320===================================================================
321Index: tests/modeltests/create_db_schema/models.py
322===================================================================
323--- tests/modeltests/create_db_schema/models.py (revision 0)
324+++ tests/modeltests/create_db_schema/models.py (revision 0)
325@@ -0,0 +1,201 @@
326+"""
327+xx. create_db_schema
328+
329+Models can have a ``create_db_schema`` attribute, which specifies
330+whether the SQL code is generated for the table on various manage.py operations
331+or not.
332+"""
333+
334+from django.db import models
335+
336+"""
337+General test strategy:
338+* All tests are numbered (01, 02, 03... etc).
339+* Each test contains three models (A, B, C, followed with the number of test),
340+  containing both indexed and non-indexed fields (to verify sql_index),
341+  usual fields (model A), foreign keys (model B) and many-to-many fields (model C).
342+  D table is generated automatically as intermediate M2M one.
343+* The normal (default; create_db_schema = True) behaviour during the manage.py
344+  operations is not thoroughly checked; it is the duty of the appropriate tests
345+  for the primary functionality of these operations.
346+  The most attention is paid to whether the create_db_schema = False
347+  disables the SQL generation properly.
348+* The intermediate table for M2M relations is not ever verified explicitly,
349+  because it is not ever marked with create_db_schema explicitly.
350+"""
351+
352+# This dictionary maps the name of the model/SQL table (like 'A01')
353+# to the boolean specifying whether this name should appear in the final SQL
354+checks = {}
355+
356+
357+"""
358+01: create_db_schema is not set.
359+    In such case, it should be equal (by default) to True,
360+    and SQL is generated for all three models.
361+"""
362+checks['A01'] = True
363+checks['B01'] = True
364+checks['C01'] = True
365+
366+class A01(models.Model):
367+    class Meta: db_table = 'A01'
368+
369+    f_a = models.TextField(db_index = True)
370+    f_b = models.IntegerField()
371+
372+class B01(models.Model):
373+    class Meta: db_table = 'B01'
374+
375+    fk_a = models.ForeignKey(A01)
376+    f_a = models.TextField(db_index = True)
377+    f_b = models.IntegerField()
378+
379+class C01(models.Model):
380+    class Meta: db_table = 'C01'
381+
382+    mm_a = models.ManyToManyField(A01, db_table = 'D01')
383+    f_a = models.TextField(db_index = True)
384+    f_b = models.IntegerField()
385+
386+
387+"""
388+02: create_db_schema is set to True.
389+    SQL is generated for all three models.
390+"""
391+checks['A02'] = True
392+checks['B02'] = True
393+checks['C02'] = True
394+
395+class A02(models.Model):
396+    class Meta:
397+        db_table = 'A02'
398+        create_db_schema = True
399+
400+    f_a = models.TextField(db_index = True)
401+    f_b = models.IntegerField()
402+
403+class B02(models.Model):
404+    class Meta:
405+        db_table = 'B02'
406+        create_db_schema = True
407+
408+    fk_a = models.ForeignKey(A02)
409+    f_a = models.TextField(db_index = True)
410+    f_b = models.IntegerField()
411+
412+class C02(models.Model):
413+    class Meta:
414+        db_table = 'C02'
415+        create_db_schema = True
416+
417+    mm_a = models.ManyToManyField(A02, db_table = 'D02')
418+    f_a = models.TextField(db_index = True)
419+    f_b = models.IntegerField()
420+
421+
422+"""
423+03: create_db_schema is set to False.
424+    SQL is NOT generated for any of the three models.
425+"""
426+checks['A03'] = False
427+checks['B03'] = False
428+checks['C03'] = False
429+
430+class A03(models.Model):
431+    class Meta:
432+        db_table = 'A03'
433+        create_db_schema = False
434+
435+    f_a = models.TextField(db_index = True)
436+    f_b = models.IntegerField()
437+
438+class B03(models.Model):
439+    class Meta:
440+        db_table = 'B03'
441+        create_db_schema = False
442+
443+    fk_a = models.ForeignKey(A03)
444+    f_a = models.TextField(db_index = True)
445+    f_b = models.IntegerField()
446+
447+class C03(models.Model):
448+    class Meta:
449+        db_table = 'C03'
450+        create_db_schema = False
451+
452+    mm_a = models.ManyToManyField(A03, db_table = 'D03')
453+    f_a = models.TextField(db_index = True)
454+    f_b = models.IntegerField()
455+
456+
457+# We will use short names for these templates
458+sql_templates = {
459+       'create table': 'CREATE TABLE "%s"',
460+       'create index': 'CREATE INDEX "%s_f_a"',
461+       'drop table': 'DROP TABLE "%s"',
462+       'delete from': 'DELETE FROM "%s"'
463+}
464+
465+def get_failed_models(arr_sql, sql_template_names):
466+    """
467+    Find the models which should not be in the SQL but they are present,
468+    or they should be in the SQL but they are missing.
469+    """
470+    txt_sql = ' '.join(arr_sql)
471+    for (model, should_be_present) in checks.iteritems():
472+        # Do we expect to see the model name in the SQL text?
473+        for sql_template_name in sql_template_names:
474+            # We are interested not in just the model name like "A01",
475+            # but in the whole string like 'CREATE TABLE "A01"'
476+            # so we apply the model name to the template
477+            # to find out the expected string
478+            expected = (sql_templates[sql_template_name])%model
479+            if ((expected in txt_sql) != should_be_present):
480+                # Our expectations failed!
481+                yield 'The string %s %s present in SQL but it %s.'%(
482+                    expected,
483+                    {False: 'is not', True: 'is'}[expected in txt_sql],
484+                    {False: 'should not be', True: 'should be'}[should_be_present]
485+                    )
486+
487+
488+__test__ = {'API_TESTS':"""
489+>>> from django.db.models import get_app
490+>>> from django.core.management.sql import *
491+>>> from django.core.management.color import no_style
492+>>> import sys
493+
494+>>> myapp = get_app('create_db_schema')
495+>>> mystyle = no_style()
496+
497+# a. Verify sql_create
498+>>> list(get_failed_models( sql_create(myapp, mystyle), ['create table'] ))
499+[]
500+
501+# b. Verify sql_delete
502+>>> list(get_failed_models( sql_delete(myapp, mystyle), ['drop table'] ))
503+[]
504+
505+# c. Verify sql_reset
506+>>> list(get_failed_models( sql_reset(myapp, mystyle), ['drop table', 'create table', 'create index'] ))
507+[]
508+
509+# d. Verify sql_flush
510+>>> # sql_flush(mystyle)
511+>>> list(get_failed_models( sql_flush(mystyle), ['delete from'] ))
512+[]
513+
514+# e. Verify sql_custom
515+# No custom data provided, should not be no output.
516+>>> sql_custom(myapp)
517+[]
518+
519+# f. Verify sql_indexes
520+>>> list(get_failed_models( sql_indexes(myapp, mystyle), ['create index'] ))
521+[]
522+
523+# g. Verify sql_all
524+>>> list(get_failed_models( sql_all(myapp, mystyle), ['create table', 'create index'] ))
525+[]
526+"""}
527Index: AUTHORS
528===================================================================
529--- AUTHORS     (revision 7027)
530+++ AUTHORS     (working copy)
531@@ -353,6 +353,7 @@
532     ymasuda@ethercube.com
533     Jarek Zgoda <jarek.zgoda@gmail.com>
534     Cheng Zhang
535+    Alexander Myodov <amyodov@gmail.com>
536 
537 A big THANK YOU goes to:
538 
539Index: docs/model-api.txt
540===================================================================
541--- docs/model-api.txt  (revision 7027)
542+++ docs/model-api.txt  (working copy)
543@@ -1051,6 +1051,28 @@
544 that aren't allowed in Python variable names -- notably, the hyphen --
545 that's OK. Django quotes column and table names behind the scenes.
546 
547+``create_db_schema``
548+--------------------
549+
550+**New in Django development version**
551+
552+Marks this model as requiring SQL operations when calling ``manage.py``::
553+
554+    create_db_schema = False
555+
556+If this isn't given, Django will use ``create_db_schema = True``
557+what means that the operations like ``manage.py sqlreset``, ``manage.py syncdb``
558+and others will regenerate the appropriate table when needed.
559+If the option is set to False, the appropriate table will not be affected with
560+any SQL operations.
561+
562+This is useful for databases where some DB tables are controlled by Django models,
563+but some other DB tables and views are created via raw SQL and should not be affected
564+by any ``manage.py`` actions.
565+Note that if the initial SQL data is provided (see `Providing initial SQL
566+data`_ below), it still will be present in the output of
567+``sqlall``/``sqlcustom`` commands.
568+
569 ``db_tablespace``
570 -----------------
571