Code

Ticket #6148: generic-db_schema-r8696.diff

File generic-db_schema-r8696.diff, 21.0 KB (added by crippledcanary, 6 years ago)

Works with postgres and mysql but testing only works on postgres due to mysqls schema implementation.

Line 
1Index: django/core/management/commands/syncdb.py
2===================================================================
3--- django/core/management/commands/syncdb.py   (revision 8696)
4+++ django/core/management/commands/syncdb.py   (working copy)
5@@ -61,6 +61,9 @@
6             app_name = app.__name__.split('.')[-2]
7             model_list = models.get_models(app)
8             for model in model_list:
9+                # Add model defined schema tables if anny
10+                if model._meta.db_schema:
11+                    tables += connection.introspection.schema_table_names(model._meta.db_schema)
12                 # Create the model's database table, if it doesn't already exist.
13                 if verbosity >= 2:
14                     print "Processing %s.%s model" % (app_name, model._meta.object_name)
15Index: django/core/management/sql.py
16===================================================================
17--- django/core/management/sql.py       (revision 8696)
18+++ django/core/management/sql.py       (working copy)
19@@ -84,6 +84,10 @@
20     references_to_delete = {}
21     app_models = models.get_models(app)
22     for model in app_models:
23+        schema = model._meta.db_schema
24+        # Find aditional tables in model defined schemas
25+        if schema:
26+            table_names += connection.introspection.get_schema_table_list(cursor, schema)
27         if cursor and connection.introspection.table_name_converter(model._meta.db_table) in table_names:
28             # The table exists, so it needs to be dropped
29             opts = model._meta
30Index: django/db/backends/__init__.py
31===================================================================
32--- django/db/backends/__init__.py      (revision 8696)
33+++ django/db/backends/__init__.py      (working copy)
34@@ -225,6 +225,13 @@
35         """
36         raise NotImplementedError()
37 
38+    def prep_db_table(self, db_schema, db_table):
39+        """
40+        Prepares and formats the table name if neccesary.
41+        Just returns the db_table if not supported
42+        """
43+        return db_table
44+
45     def random_function_sql(self):
46         """
47         Returns a SQL expression that returns a random value.
48@@ -386,6 +393,30 @@
49         cursor = self.connection.cursor()
50         return self.get_table_list(cursor)
51 
52+    def schema_name_converter(self, name):
53+        """Apply a conversion to the name for the purposes of comparison.
54+
55+        The default schema name converter is for case sensitive comparison.
56+        """
57+        return name
58+
59+    def get_schema_list(self, cursor):
60+        "Returns a list of schemas that exist in the database"
61+        return []
62+   
63+    def get_schema_table_list(self, cursor, schema):
64+        "Returns a list of tables in a specific schema"
65+        return []
66+       
67+    def schema_names(self):
68+        cursor = self.connection.cursor()
69+        return self.get_schema_list(cursor)
70+   
71+    def schema_table_names(self, schema):
72+        "Returns a list of names of all tables that exist in the database schema."
73+        cursor = self.connection.cursor()
74+        return self.get_schema_table_list(cursor, schema)
75+
76     def django_table_names(self, only_existing=False):
77         """
78         Returns a list of all table names that have associated Django models and
79Index: django/db/backends/creation.py
80===================================================================
81--- django/db/backends/creation.py      (revision 8696)
82+++ django/db/backends/creation.py      (working copy)
83@@ -25,6 +25,17 @@
84     def __init__(self, connection):
85         self.connection = connection
86 
87+    def default_schema(self):
88+        return ""
89+
90+    def sql_create_schema(self, schema, style):
91+        """"
92+        Returns the SQL required to create a single schema
93+        """
94+        qn = self.connection.ops.quote_name
95+        output = "%s %s;" % (style.SQL_KEYWORD('CREATE SCHEMA'), qn(schema))
96+        return output
97+
98     def sql_create_model(self, model, style, known_models=set()):
99         """
100         Returns the SQL required to create a single model, as a tuple of:
101@@ -122,6 +133,10 @@
102                 r_col = f.column
103                 table = opts.db_table
104                 col = opts.get_field(f.rel.field_name).column
105+                # Add schema if we are related to a model in different schema
106+                # and we are not in a different schema ourselfs
107+                if rel_opts.db_schema and not opts.db_schema:
108+                    table =  "%s.%s" % (self.default_schema(), table)
109                 # For MySQL, r_name must be unique in the first 64 characters.
110                 # So we are careful with character usage here.
111                 r_name = '%s_refs_%s_%x' % (r_col, col, abs(hash((r_table, table))))
112@@ -243,8 +258,13 @@
113                     tablespace_sql = ''
114             else:
115                 tablespace_sql = ''
116+            # Use original db_table in index name if schema is provided
117+            if model._meta.db_schema:
118+                index_table_name = model._meta._db_table
119+            else:
120+                index_table_name = model._meta.db_table
121             output = [style.SQL_KEYWORD('CREATE INDEX') + ' ' +
122-                style.SQL_TABLE(qn('%s_%s' % (model._meta.db_table, f.column))) + ' ' +
123+                style.SQL_TABLE(qn('%s_%s' % (index_table_name, f.column))) + ' ' +
124                 style.SQL_KEYWORD('ON') + ' ' +
125                 style.SQL_TABLE(qn(model._meta.db_table)) + ' ' +
126                 "(%s)" % style.SQL_FIELD(qn(f.column)) +
127@@ -253,6 +273,14 @@
128             output = []
129         return output
130 
131+    def sql_destroy_schema(self, schema, style):
132+        """"
133+        Returns the SQL required to create a single schema
134+        """
135+        qn = self.connection.ops.quote_name
136+        output = "%s %s CASCADE;" % (style.SQL_KEYWORD('DROP SCHEMA IF EXISTS'), qn(schema))
137+        return output
138+
139     def sql_destroy_model(self, model, references_to_delete, style):
140         "Return the DROP TABLE and restraint dropping statements for a single model"
141         # Drop the table now
142@@ -324,8 +352,55 @@
143 
144         return test_database_name
145 
146+    def _create_test_schemas(self, verbosity, schemas, cursor):
147+        from django.core.management.color import color_style
148+        style = color_style()
149+        for schema in schemas:
150+            if verbosity >= 1:
151+                print "Creating schema %s" % schema
152+            cursor.execute(self.sql_create_schema(schema, style))
153+
154+    def _destroy_test_schemas(self, verbosity, schemas, cursor):
155+        from django.core.management.color import color_style
156+        style = color_style()
157+        for schema in schemas:
158+            if verbosity >= 1:
159+                print "Destroying schema %s" % schema
160+                cursor.execute(self.sql_destroy_schema(schema, style))
161+            if verbosity >= 1:
162+                print "Schema %s destroyed" % schema
163+
164+    def _get_schemas(self, apps):
165+        from django.db import models
166+        schemas = set()
167+        for app in apps:
168+            app_models = models.get_models(app)
169+            for model in app_models:
170+                schema = model._meta.db_schema
171+                if not schema or schema in schemas:
172+                    continue
173+                schemas.add(schema)
174+        return schemas
175+
176+    def _get_app_with_schemas(self):
177+        from django.db import models
178+        apps = models.get_apps()
179+        schema_apps = set()
180+        for app in apps:
181+            app_models = models.get_models(app)
182+            for model in app_models:
183+                schema = model._meta.db_schema
184+                if not schema or app in schema_apps:
185+                    continue
186+                schema_apps.add(app)
187+                continue
188+               
189+        return schema_apps
190+
191     def _create_test_db(self, verbosity, autoclobber):
192         "Internal implementation - creates the test db tables."
193+        schema_apps = self._get_app_with_schemas()
194+        schemas = self._get_schemas(schema_apps)
195         suffix = self.sql_table_creation_suffix()
196 
197         if settings.TEST_DATABASE_NAME:
198@@ -342,6 +417,12 @@
199         self.set_autocommit()
200         try:
201             cursor.execute("CREATE DATABASE %s %s" % (qn(test_database_name), suffix))
202+            #Connect to the new database to create schemas in it
203+            self.connection.close()
204+            settings.DATABASE_NAME = test_database_name
205+            cursor = self.connection.cursor()
206+            self.set_autocommit()
207+            self._create_test_schemas(verbosity, schemas, cursor)
208         except Exception, e:
209             sys.stderr.write("Got an error creating the test database: %s\n" % e)
210             if not autoclobber:
211@@ -350,10 +431,17 @@
212                 try:
213                     if verbosity >= 1:
214                         print "Destroying old test database..."
215+                    self._destroy_test_schemas(verbosity, schemas, cursor)
216                     cursor.execute("DROP DATABASE %s" % qn(test_database_name))
217                     if verbosity >= 1:
218                         print "Creating test database..."
219                     cursor.execute("CREATE DATABASE %s %s" % (qn(test_database_name), suffix))
220+                    #Connect to the new database to create schemas in it
221+                    self.connection.close()
222+                    settings.DATABASE_NAME = test_database_name
223+                    cursor = self.connection.cursor()
224+                    self.set_autocommit()
225+                    self._create_test_schemas(verbosity, schemas, cursor)
226                 except Exception, e:
227                     sys.stderr.write("Got an error recreating the test database: %s\n" % e)
228                     sys.exit(2)
229Index: django/db/backends/mysql/base.py
230===================================================================
231--- django/db/backends/mysql/base.py    (revision 8696)
232+++ django/db/backends/mysql/base.py    (working copy)
233@@ -142,8 +142,12 @@
234     def quote_name(self, name):
235         if name.startswith("`") and name.endswith("`"):
236             return name # Quoting once is enough.
237-        return "`%s`" % name
238+        # add support for tablenames passed that also have their schema in their name
239+        return "`%s`" % name.replace('.','`.`')
240 
241+    def prep_db_table(self, db_schema, db_table):
242+        return "%s.%s" % (db_schema, db_table)
243+
244     def random_function_sql(self):
245         return 'RAND()'
246 
247Index: django/db/backends/mysql/creation.py
248===================================================================
249--- django/db/backends/mysql/creation.py        (revision 8696)
250+++ django/db/backends/mysql/creation.py        (working copy)
251@@ -65,4 +65,23 @@
252                 field.rel.to._meta.db_table, field.rel.to._meta.pk.column)
253             ]
254         return table_output, deferred
255-       
256\ No newline at end of file
257+
258+    def default_schema(self):
259+        return settings.DATABASE_NAME
260+
261+    def sql_create_schema(self, schema, style):
262+        """
263+        Returns the SQL required to create a single schema.
264+        In MySQL schemas are synonymous to databases
265+        """
266+        qn = self.connection.ops.quote_name
267+        output = "%s %s;" % (style.SQL_KEYWORD('CREATE DATABASE'), qn(schema))
268+        return output
269+
270+    def sql_destroy_schema(self, schema, style):
271+        """"
272+        Returns the SQL required to create a single schema
273+        """
274+        qn = self.connection.ops.quote_name
275+        output = "%s %s;" % (style.SQL_KEYWORD('DROP DATABASE IF EXISTS'), qn(schema))
276+        return output
277\ No newline at end of file
278Index: django/db/backends/mysql/introspection.py
279===================================================================
280--- django/db/backends/mysql/introspection.py   (revision 8696)
281+++ django/db/backends/mysql/introspection.py   (working copy)
282@@ -33,6 +33,14 @@
283         cursor.execute("SHOW TABLES")
284         return [row[0] for row in cursor.fetchall()]
285 
286+    def get_schema_list(self, cursor):
287+        cursor.execute("SHOW SCHEMAS")
288+        return [row[0] for row in cursor.fetchall()]
289+
290+    def get_schema_table_list(self, cursor, schema):
291+        cursor.execute("SHOW TABLES FROM %s" % self.connection.ops.quote_name(schema))
292+        return [schema + "." + row[0] for row in cursor.fetchall()]
293+
294     def get_table_description(self, cursor, table_name):
295         "Returns a description of the table, with the DB-API cursor.description interface."
296         cursor.execute("SELECT * FROM %s LIMIT 1" % self.connection.ops.quote_name(table_name))
297Index: django/db/backends/postgresql/introspection.py
298===================================================================
299--- django/db/backends/postgresql/introspection.py      (revision 8696)
300+++ django/db/backends/postgresql/introspection.py      (working copy)
301@@ -29,6 +29,24 @@
302                 AND pg_catalog.pg_table_is_visible(c.oid)""")
303         return [row[0] for row in cursor.fetchall()]
304 
305+    def get_schema_list(self, cursor):
306+        cursor.execute("""
307+            SELECT DISTINCT n.nspname
308+            FROM pg_catalog.pg_class c
309+            LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
310+            WHERE c.relkind IN ('r', 'v', '')
311+            AND n.nspname NOT IN ('pg_catalog', 'pg_toast', 'information_schema')""")   
312+        return [row[0] for row in cursor.fetchall()]
313+
314+    def get_schema_table_list(self, cursor, schema):
315+        cursor.execute("""
316+            SELECT c.relname
317+            FROM pg_catalog.pg_class c
318+            LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
319+            WHERE c.relkind IN ('r', 'v', '')
320+                AND n.nspname = '%s'""" % schema)
321+        return [schema + "." + row[0] for row in cursor.fetchall()]
322+
323     def get_table_description(self, cursor, table_name):
324         "Returns a description of the table, with the DB-API cursor.description interface."
325         cursor.execute("SELECT * FROM %s LIMIT 1" % self.connection.ops.quote_name(table_name))
326Index: django/db/backends/postgresql/operations.py
327===================================================================
328--- django/db/backends/postgresql/operations.py (revision 8696)
329+++ django/db/backends/postgresql/operations.py (working copy)
330@@ -55,7 +55,8 @@
331         return '%s'
332 
333     def last_insert_id(self, cursor, table_name, pk_name):
334-        cursor.execute("SELECT CURRVAL('\"%s_%s_seq\"')" % (table_name, pk_name))
335+        # add support for tablenames passed that also have their schema in their name
336+        cursor.execute(("SELECT CURRVAL('\"%s_%s_seq\"')" % (table_name, pk_name)).replace('.', '"."'))
337         return cursor.fetchone()[0]
338 
339     def no_limit_value(self):
340@@ -64,8 +65,12 @@
341     def quote_name(self, name):
342         if name.startswith('"') and name.endswith('"'):
343             return name # Quoting once is enough.
344-        return '"%s"' % name
345+        # add support for tablenames passed that also have their schema in their name
346+        return '"%s"' % name.replace('.','"."')
347 
348+    def prep_db_table(self, db_schema, db_table):
349+        return "%s.%s" % (db_schema, db_table)
350+
351     def sql_flush(self, style, tables, sequences):
352         if tables:
353             if self.postgres_version[0] >= 8 and self.postgres_version[1] >= 1:
354Index: django/db/models/options.py
355===================================================================
356--- django/db/models/options.py (revision 8696)
357+++ django/db/models/options.py (working copy)
358@@ -21,7 +21,7 @@
359 DEFAULT_NAMES = ('verbose_name', 'db_table', 'ordering',
360                  'unique_together', 'permissions', 'get_latest_by',
361                  'order_with_respect_to', 'app_label', 'db_tablespace',
362-                 'abstract')
363+                 'abstract', 'db_schema')
364 
365 class Options(object):
366     def __init__(self, meta, app_label=None):
367@@ -29,6 +29,7 @@
368         self.module_name, self.verbose_name = None, None
369         self.verbose_name_plural = None
370         self.db_table = ''
371+        self.db_schema = ''
372         self.ordering = []
373         self.unique_together =  []
374         self.permissions =  []
375@@ -95,6 +96,14 @@
376             self.db_table = "%s_%s" % (self.app_label, self.module_name)
377             self.db_table = truncate_name(self.db_table, connection.ops.max_name_length())
378 
379+        # Patch db_table with the schema if provided and allowed
380+        if self.db_schema:
381+            # Store original db_table in a save place first
382+            self._db_table = self.db_table
383+            self.db_table = connection.ops.prep_db_table(self.db_schema, self.db_table)
384+            # If no changes were done then backend don't support schemas
385+            if self._db_table == self.db_table:
386+                self.db_schema = ''
387 
388     def _prepare(self, model):
389         if self.order_with_respect_to:
390Index: docs/ref/models/options.txt
391===================================================================
392--- docs/ref/models/options.txt (revision 8696)
393+++ docs/ref/models/options.txt (working copy)
394@@ -42,6 +42,21 @@
395 aren't allowed in Python variable names -- notably, the hyphen -- that's OK.
396 Django quotes column and table names behind the scenes.
397 
398+``db_schema``
399+-----------------
400+
401+**New in Django development version**
402+
403+The name of the database schema to use for the model. If the backend
404+doesn't support multiple schemas, this options is ignored.
405+
406+If this is used Django will prefix any table names with the schema name.
407+For example MySQL Django would use ``db_schema + '.' + db_table``.
408+Be aware that postgres supports different schemas within the database.
409+MySQL solves the same thing by treating it as just another database.
410+
411+
412+
413 ``db_tablespace``
414 -----------------
415 
416Index: docs/topics/db/models.txt
417===================================================================
418--- docs/topics/db/models.txt   (revision 8696)
419+++ docs/topics/db/models.txt   (working copy)
420@@ -594,7 +594,8 @@
421             verbose_name_plural = "oxen"
422 
423 Model metadata is "anything that's not a field", such as ordering options
424-(:attr:`~Options.ordering`), database table name (:attr:`~Options.db_table`), or
425+(:attr:`~Options.ordering`), database table name (:attr:`~Options.db_table`),
426+(:attr:`~Options.db_schema`) custom schema for the tables, or
427 human-readable singular and plural names (:attr:`~Options.verbose_name` and
428 :attr:`~Options.verbose_name_plural`). None are required, and adding ``class
429 Meta`` to a model is completely optional.
430Index: tests/modeltests/schemas/__init__.py
431===================================================================
432--- tests/modeltests/schemas/__init__.py        (revision 0)
433+++ tests/modeltests/schemas/__init__.py        (revision 0)
434@@ -0,0 +1 @@
435+#
436Index: tests/modeltests/schemas/models.py
437===================================================================
438--- tests/modeltests/schemas/models.py  (revision 0)
439+++ tests/modeltests/schemas/models.py  (revision 0)
440@@ -0,0 +1,98 @@
441+# coding: utf-8
442+
443+from django.db import models
444+
445+
446+class Blog(models.Model):
447+    "Model in default schema"
448+    name = models.CharField(max_length=50)
449+
450+   
451+class Entry(models.Model):
452+    "Model in custom schema that reference the default"
453+    blog = models.ForeignKey(Blog)   
454+    title = models.CharField(max_length=50)
455+   
456+    class Meta:
457+        "using custom db_table as well"
458+        db_table='schema_blog_entries'
459+        db_schema = 'test_schema'
460+       
461+
462+class Comment(models.Model):
463+    "Model in the custom schema that references Entry in the same schema"
464+    entry = models.ForeignKey(Entry)
465+    text = models.CharField(max_length=50)
466+   
467+    class Meta:
468+        db_schema = 'test_schema'
469+
470+__test__ = {'API_TESTS': """
471+
472+#Test with actual data
473+# Nothing in there yet
474+>>> Blog.objects.all()
475+[]
476+
477+# Create a blog
478+>>> b = Blog(name='Test')
479+>>> b.save()
480+
481+# Verify that we got an ID
482+>>> b.id
483+1
484+
485+# Create entry
486+>>> e = Entry(blog=b, title='Test entry')
487+>>> e.save()
488+>>> e.id
489+1
490+
491+# Create Comments
492+>>> c1 = Comment(entry=e, text='nice entry')
493+>>> c1.save()
494+>>> c2 = Comment(entry=e, text='really like it')
495+>>> c2.save()
496+
497+#Retrieve the stuff again.
498+>>> b2 = Blog.objects.get(id=b.id)
499+>>> b==b2
500+True
501+
502+>>> b2.entry_set.all()
503+[<Entry: Entry object>]
504+
505+>>> from django.conf import settings
506+>>> from django.db import connection, models
507+
508+# Test if we support schemas and can find the table if so
509+>>> if e._meta.db_schema:
510+...     tables = connection.introspection.schema_table_names(e._meta.db_schema)
511+... else:
512+...     tables = connection.introspection.table_names()
513+>>> if connection.introspection.table_name_converter(e._meta.db_table) in tables:
514+...     print "ok"
515+... else:
516+...     print "schema=" + e._meta.db_schema
517+...     print tables
518+ok
519+
520+# Test that all but sqlite3 backend suports schema and doesn't drop it.
521+# Oracle is not tested
522+>>> if settings.DATABASE_ENGINE != 'sqlite3' and settings.DATABASE_ENGINE != 'oracle':
523+...     if e._meta.db_schema != 'test_schema':
524+...         print "shouldn't drop or modify schema"
525+
526+>>> from django.core.management.sql import *
527+>>> from django.core.management.color import color_style
528+
529+>>> style = color_style()
530+>>> app = models.get_app('schemas')
531+
532+# Get the sql_create sequence
533+>>> a = sql_create(app, style)
534+
535+
536+#Done
537+"""
538+}