=== django/test/utils.py ================================================================== --- django/test/utils.py (/mirror/django/trunk) (revision 4246) +++ django/test/utils.py (/local/django/multidb) (revision 4246) @@ -1,6 +1,6 @@ import sys, time from django.conf import settings -from django.db import connection, get_creation_module +from django.db import connection, connections, get_creation_module from django.core import mail from django.core.management import call_command from django.dispatch import dispatcher @@ -106,6 +106,7 @@ # in-memory database. if settings.DATABASE_ENGINE == "sqlite3": TEST_DATABASE_NAME = ":memory:" +## TEST_DATABASE_NAME = settings.TEST_DATABASE_NAME else: suffix = { 'postgresql': get_postgresql_create_suffix, @@ -148,6 +149,7 @@ connection.close() settings.DATABASE_NAME = TEST_DATABASE_NAME + settings.OTHER_DATABASES = settings.TEST_OTHER_DATABASES call_command('syncdb', verbosity=verbosity, interactive=False) @@ -161,7 +163,8 @@ return TEST_DATABASE_NAME -def destroy_test_db(old_database_name, verbosity=1): +def destroy_test_db(old_database_name, old_databases, verbosity=1): + # If the database wants to drop the test DB itself, let it creation_module = get_creation_module() if hasattr(creation_module, "destroy_test_db"): @@ -175,7 +178,12 @@ if verbosity >= 1: print "Destroying test database..." connection.close() +## for cnx in connections.keys(): +## connections[cnx].close() +## connections.reset() TEST_DATABASE_NAME = settings.DATABASE_NAME + if verbosity >= 2: + print "Closed connections to %s" % TEST_DATABASE_NAME settings.DATABASE_NAME = old_database_name if settings.DATABASE_ENGINE != "sqlite3": @@ -184,3 +192,11 @@ time.sleep(1) # To avoid "database is being accessed by other users" errors. cursor.execute("DROP DATABASE %s" % connection.ops.quote_name(TEST_DATABASE_NAME)) connection.close() +## settings.OTHER_DATABASES = old_databases +## for cnx in connections.keys(): +## try: +## cursor = connections[cnx].connection.cursor() +## except (KeyboardInterrupt, SystemExit): +## raise +## except: +## pass === django/db/models/base.py ================================================================== --- django/db/models/base.py (/mirror/django/trunk) (revision 4246) +++ django/db/models/base.py (/local/django/multidb) (revision 4246) @@ -6,7 +6,7 @@ from django.db.models.fields.related import OneToOneRel, ManyToOneRel from django.db.models.query import delete_objects from django.db.models.options import Options, AdminOptions -from django.db import connection, transaction +from django.db import transaction from django.db.models import signals from django.db.models.loading import register_models, get_model from django.dispatch import dispatcher @@ -209,6 +209,8 @@ def save(self, raw=False): dispatcher.send(signal=signals.pre_save, sender=self.__class__, instance=self) + db = self.__class__._default_manager.db + connection = db.connection non_pks = [f for f in self._meta.fields if not f.primary_key] cursor = connection.cursor() @@ -227,7 +229,7 @@ self._meta.pk.get_db_prep_lookup('exact', pk_val)) # If it does already exist, do an UPDATE. if cursor.fetchone(): - db_values = [f.get_db_prep_save(raw and getattr(self, f.attname) or f.pre_save(self, False)) for f in non_pks] + db_values = [f.get_db_prep_save(self, raw and getattr(self, f.attname) or f.pre_save(self, False)) for f in non_pks] if db_values: cursor.execute("UPDATE %s SET %s WHERE %s=%%s" % \ (qn(self._meta.db_table), @@ -238,11 +240,11 @@ record_exists = False if not pk_set or not record_exists: field_names = [qn(f.column) for f in self._meta.fields if not isinstance(f, AutoField)] - db_values = [f.get_db_prep_save(raw and getattr(self, f.attname) or f.pre_save(self, True)) for f in self._meta.fields if not isinstance(f, AutoField)] + db_values = [f.get_db_prep_save(self, raw and getattr(self, f.attname) or f.pre_save(self, True)) for f in self._meta.fields if not isinstance(f, AutoField)] # If the PK has been manually set, respect that. if pk_set: field_names += [f.column for f in self._meta.fields if isinstance(f, AutoField)] - db_values += [f.get_db_prep_save(raw and getattr(self, f.attname) or f.pre_save(self, True)) for f in self._meta.fields if isinstance(f, AutoField)] + db_values += [f.get_db_prep_save(self, raw and getattr(self, f.attname) or f.pre_save(self, True)) for f in self._meta.fields if isinstance(f, AutoField)] placeholders = ['%s'] * len(field_names) if self._meta.order_with_respect_to: field_names.append(qn('_order')) @@ -264,7 +266,7 @@ connection.ops.pk_default_value())) if self._meta.has_auto_field and not pk_set: setattr(self, self._meta.pk.attname, connection.ops.last_insert_id(cursor, self._meta.db_table, self._meta.pk.column)) - transaction.commit_unless_managed() + transaction.commit_unless_managed([connection]) # Run any post-save hooks. dispatcher.send(signal=signals.post_save, sender=self.__class__, @@ -336,6 +338,8 @@ return force_unicode(dict(field.choices).get(value, value), strings_only=True) def _get_next_or_previous_by_FIELD(self, field, is_next, **kwargs): + db = self.__class__._default_manager.db + connection = db.connection qn = connection.ops.quote_name op = is_next and '>' or '<' where = '(%s %s %%s OR (%s = %%s AND %s.%s %s %%s))' % \ @@ -351,6 +355,8 @@ raise self.DoesNotExist, "%s matching query does not exist." % self.__class__._meta.object_name def _get_next_or_previous_in_order(self, is_next): + db = self.__class__._default_manager.db + connection = db.connection qn = connection.ops.quote_name cachename = "__%s_order_cache" % is_next if not hasattr(self, cachename): @@ -441,6 +447,8 @@ # ORDERING METHODS ######################### def method_set_order(ordered_obj, self, id_list): + db = ordered_obj._default_manager.db + connection = db.connection qn = connection.ops.quote_name cursor = connection.cursor() # Example: "UPDATE poll_choices SET _order = %s WHERE poll_id = %s AND id = %s" @@ -450,9 +458,11 @@ qn(ordered_obj._meta.pk.column)) rel_val = getattr(self, ordered_obj._meta.order_with_respect_to.rel.field_name) cursor.executemany(sql, [(i, rel_val, j) for i, j in enumerate(id_list)]) - transaction.commit_unless_managed() + transaction.commit_unless_managed([connection]) def method_get_order(ordered_obj, self): + db = ordered_obj._default_manager.db + connection = db.connection qn = connection.ops.quote_name cursor = connection.cursor() # Example: "SELECT id FROM poll_choices WHERE poll_id = %s ORDER BY _order" === django/db/models/manager.py ================================================================== --- django/db/models/manager.py (/mirror/django/trunk) (revision 4246) +++ django/db/models/manager.py (/local/django/multidb) (revision 4246) @@ -1,6 +1,7 @@ -from django.db.models.query import QuerySet, EmptyQuerySet +from django.db import ConnectionInfoDescriptor +from django.db.models.query import _QuerySet, EmptyQuerySet from django.dispatch import dispatcher -from django.db.models import signals +from django.db.models import signals, get_apps, get_models from django.db.models.fields import FieldDoesNotExist def ensure_default_manager(sender): @@ -13,13 +14,17 @@ except FieldDoesNotExist: pass cls.add_to_class('objects', Manager()) + elif cls._default_manager.model != cls: + # cls is an inherited model; don't want the parent manager + cls.add_to_class('objects', Manager()) dispatcher.connect(ensure_default_manager, signal=signals.class_prepared) class Manager(object): # Tracks each time a Manager instance is created. Used to retain order. creation_counter = 0 - + db = ConnectionInfoDescriptor() + def __init__(self): super(Manager, self).__init__() # Increase the creation counter, and save our local copy. @@ -31,9 +36,11 @@ # TODO: Use weakref because of possible memory leak / circular reference. self.model = model setattr(model, name, ManagerDescriptor(self)) - if not hasattr(model, '_default_manager') or self.creation_counter < model._default_manager.creation_counter: + if not hasattr(model, '_default_manager') \ + or self.creation_counter < model._default_manager.creation_counter \ + or model._default_manager.model != model: model._default_manager = self - + ####################### # PROXIES TO QUERYSET # ####################### @@ -45,7 +52,11 @@ """Returns a new QuerySet object. Subclasses can override this method to easily customize the behavior of the Manager. """ - return QuerySet(self.model) + if self.db.connection.features.uses_custom_queryset: + QS = self.db.connection.ops.query_set_class(_QuerySet) + return QS(self.model) + else: + return _QuerySet(self.model) def none(self): return self.get_empty_query_set() === django/db/models/options.py ================================================================== --- django/db/models/options.py (/mirror/django/trunk) (revision 4246) +++ django/db/models/options.py (/local/django/multidb) (revision 4246) @@ -1,4 +1,5 @@ from django.conf import settings +from django.db import connection_info, connections from django.db.models.related import RelatedObject from django.db.models.fields.related import ManyToManyRel from django.db.models.fields import AutoField, FieldDoesNotExist @@ -9,6 +10,7 @@ from django.utils.encoding import force_unicode, smart_str from bisect import bisect import re +import weakref # Calculate the verbose_name by converting from InitialCaps to "lowercase with spaces". get_verbose_name = lambda class_name: re.sub('(((?<=[a-z])[A-Z])|([A-Z](?![A-Z]|$)))', ' \\1', class_name).lower().strip() @@ -91,6 +93,9 @@ self.db_table = "%s_%s" % (self.app_label, self.module_name) self.db_table = truncate_name(self.db_table, connection.ops.max_name_length()) + # Keep a weakref to my model, for access to managers and such + self._model = weakref.ref(model) + def add_field(self, field): # Insert the given field in the order in which it was created, using # the "creation_counter" attribute of the field. @@ -130,6 +135,12 @@ return f raise FieldDoesNotExist, '%s has no field named %r' % (self.object_name, name) + def get_default_manager(self): + model = self._model() + if model is None: + raise ReferenceError("Model no longer available in %s" % self) + return model._default_manager + def get_order_sql(self, table_prefix=''): "Returns the full 'ORDER BY' clause for this object, according to self.ordering." if not self.ordering: return '' === django/db/models/fields/__init__.py ================================================================== --- django/db/models/fields/__init__.py (/mirror/django/trunk) (revision 4246) +++ django/db/models/fields/__init__.py (/local/django/multidb) (revision 4246) @@ -201,7 +201,7 @@ "Returns field's value just before saving." return getattr(model_instance, self.attname) - def get_db_prep_save(self, value): + def get_db_prep_save(self, model_instance, value): "Returns field's value prepared for saving into a database." return value @@ -536,7 +536,7 @@ else: return self.editable or self.auto_now or self.auto_now_add - def get_db_prep_save(self, value): + def get_db_prep_save(self, model_instance, value): # Casts dates into string format for entry into database. if value is not None: try: @@ -545,7 +545,7 @@ # If value is already a string it won't have a strftime method, # so we'll just let it pass through. pass - return Field.get_db_prep_save(self, value) + return Field.get_db_prep_save(self, model_instance, value) def get_manipulator_field_objs(self): return [oldforms.DateField] @@ -578,7 +578,17 @@ except ValueError: raise validators.ValidationError, _('Enter a valid date/time in YYYY-MM-DD HH:MM format.') - def get_db_prep_save(self, value): + def pre_save(self, model_instance, add): + value = super(DateTimeField, self).pre_save(model_instance, add) + if value is not None: + # MySQL will throw a warning if microseconds are given, because it + # doesn't support microseconds. + settings = model_instance._default_manager.db.connection.settings + if settings.DATABASE_ENGINE == 'mysql' and hasattr(value, 'microsecond'): + value = value.replace(microsecond=0) + return value + + def get_db_prep_save(self, model_instance, value): # Casts dates into string format for entry into database. if value is not None: # MySQL will throw a warning if microseconds are given, because it @@ -586,7 +596,7 @@ if settings.DATABASE_ENGINE == 'mysql' and hasattr(value, 'microsecond'): value = value.replace(microsecond=0) value = smart_unicode(value) - return Field.get_db_prep_save(self, value) + return Field.get_db_prep_save(self, model_instance, value) def get_db_prep_lookup(self, lookup_type, value): if lookup_type == 'range': @@ -660,10 +670,10 @@ return u"%.*f" % (self.decimal_places, value) - def get_db_prep_save(self, value): + def get_db_prep_save(self, model_instance, value): if value is not None: value = self._format(value) - return super(DecimalField, self).get_db_prep_save(value) + return super(DecimalField, self).get_db_prep_save(model_instance, value) def get_db_prep_lookup(self, lookup_type, value): if lookup_type == 'range': @@ -986,11 +996,12 @@ else: return super(TimeField, self).pre_save(model_instance, add) - def get_db_prep_save(self, value): + def get_db_prep_save(self, model_instance, value): # Casts dates into string format for entry into database. if value is not None: # MySQL will throw a warning if microseconds are given, because it # doesn't support microseconds. + settings = model_instance._default_manager.db.connection.settings if settings.DATABASE_ENGINE == 'mysql' and hasattr(value, 'microsecond'): value = value.replace(microsecond=0) if settings.DATABASE_ENGINE == 'oracle': @@ -1002,7 +1013,7 @@ value = datetime.datetime(*(time.strptime(value, '%H:%M:%S')[:6])) else: value = smart_unicode(value) - return Field.get_db_prep_save(self, value) + return Field.get_db_prep_save(self, model_instance, value) def get_manipulator_field_objs(self): return [oldforms.TimeField] === django/db/models/fields/related.py ================================================================== --- django/db/models/fields/related.py (/mirror/django/trunk) (revision 4246) +++ django/db/models/fields/related.py (/local/django/multidb) (revision 4246) @@ -1,4 +1,4 @@ -from django.db import connection, transaction +from django.db import transaction from django.db.models import signals, get_model from django.db.models.fields import AutoField, Field, IntegerField, PositiveIntegerField, PositiveSmallIntegerField, get_ul_class from django.db.models.related import RelatedObject @@ -319,6 +319,7 @@ # source_col_name: the PK colname in join_table for the source object # target_col_name: the PK colname in join_table for the target object # *objs - objects to add. Either object instances, or primary keys of object instances. + connection = self.model._default_manager.db.connection # If there aren't any objects, there is nothing to do. if objs: @@ -343,12 +344,13 @@ cursor.execute("INSERT INTO %s (%s, %s) VALUES (%%s, %%s)" % \ (self.join_table, source_col_name, target_col_name), [self._pk_val, obj_id]) - transaction.commit_unless_managed() + transaction.commit_unless_managed(connection) def _remove_items(self, source_col_name, target_col_name, *objs): # source_col_name: the PK colname in join_table for the source object # target_col_name: the PK colname in join_table for the target object # *objs - objects to remove + connection = self.model._default_manager.db.connection # If there aren't any objects, there is nothing to do. if objs: @@ -365,15 +367,16 @@ (self.join_table, source_col_name, target_col_name, ",".join(['%s'] * len(old_ids))), [self._pk_val] + list(old_ids)) - transaction.commit_unless_managed() + transaction.commit_unless_managed(connection) def _clear_items(self, source_col_name): # source_col_name: the PK colname in join_table for the source object + connection = self.model._default_manager.db.connection cursor = connection.cursor() cursor.execute("DELETE FROM %s WHERE %s = %%s" % \ (self.join_table, source_col_name), [self._pk_val]) - transaction.commit_unless_managed() + transaction.commit_unless_managed(connection) return ManyRelatedManager @@ -397,7 +400,7 @@ superclass = rel_model._default_manager.__class__ RelatedManager = create_many_related_manager(superclass) - qn = connection.ops.quote_name + qn = rel_model._default_manager.db.connection.ops.quote_name manager = RelatedManager( model=rel_model, core_filters={'%s__pk' % self.related.field.name: instance._get_pk_val()}, @@ -438,7 +441,7 @@ superclass = rel_model._default_manager.__class__ RelatedManager = create_many_related_manager(superclass) - qn = connection.ops.quote_name + qn = rel_model._default_manager.db.connection.ops.quote_name manager = RelatedManager( model=rel_model, core_filters={'%s__pk' % self.field.related_query_name(): instance._get_pk_val()}, @@ -519,11 +522,11 @@ else: return [oldforms.IntegerField] - def get_db_prep_save(self, value): + def get_db_prep_save(self, model_instance, value): if value == '' or value == None: return None else: - return self.rel.get_related_field().get_db_prep_save(value) + return self.rel.get_related_field().get_db_prep_save(model_instance, value) def flatten_data(self, follow, obj=None): if not obj: === django/db/models/query.py ================================================================== --- django/db/models/query.py (/mirror/django/trunk) (revision 4246) +++ django/db/models/query.py (/local/django/multidb) (revision 4246) @@ -77,11 +77,11 @@ output.append('%s%s ASC' % (prefix, qn(orderfield2column(f, opts)))) return ', '.join(output) -def quote_only_if_word(word): +def quote_only_if_word(word, qn): if re.search('\W', word): # Don't quote if there are spaces or non-word chars. return word else: - return connection.ops.quote_name(word) + return qn(word) class _QuerySet(object): "Represents a lazy database lookup for a set of objects" @@ -184,8 +184,7 @@ # self._select is a dictionary, and dictionaries' key order is # undefined, so we convert it to a list of tuples. extra_select = self._select.items() - - cursor = connection.cursor() + cursor = self.model._default_manager.db.connection.cursor() cursor.execute("SELECT " + (self._distinct and "DISTINCT " or "") + ",".join(select) + sql, params) fill_cache = self._select_related @@ -219,7 +218,9 @@ """ if self._result_cache is not None: return len(self._result_cache) - + #from multi-db + connection = self.model._default_manager.db.connection + counter = self._clone() counter._order_by = () counter._select_related = False @@ -306,6 +307,7 @@ Returns a dictionary mapping each of the given IDs to the object with that ID. """ + connection = self.model._default_manager.db.connection assert self._limit is None and self._offset is None, \ "Cannot use 'limit' or 'offset' with in_bulk" assert isinstance(id_list, (tuple, list)), "in_bulk() must be provided with a list of IDs." @@ -483,12 +485,12 @@ return self._result_cache def _get_sql_clause(self): + connection = self.model._default_manager.db.connection qn = connection.ops.quote_name opts = self.model._meta - # Construct the fundamental parts of the query: SELECT X FROM Y WHERE Z. select = ["%s.%s" % (qn(opts.db_table), qn(f.column)) for f in opts.fields] - tables = [quote_only_if_word(t) for t in self._tables] + tables = [quote_only_if_word(t, qn) for t in self._tables] joins = SortedDict() where = self._where[:] params = self._params[:] @@ -508,7 +510,7 @@ # Add any additional SELECTs. if self._select: - select.extend(['(%s) AS %s' % (quote_only_if_word(s[1]), qn(s[0])) for s in self._select.items()]) + select.extend(['(%s) AS %s' % (quote_only_if_word(s[1], qn), qn(s[0])) for s in self._select.items()]) # Start composing the body of the SQL statement. sql = [" FROM", qn(opts.db_table)] @@ -565,6 +567,7 @@ return select, " ".join(sql), params # Use the backend's QuerySet class if it defines one. Otherwise, use _QuerySet. +# This is now in manager.py if connection.features.uses_custom_queryset: QuerySet = connection.ops.query_set_class(_QuerySet) else: @@ -582,6 +585,9 @@ except EmptyResultSet: raise StopIteration + #from multi-db + db = self.model._default_manager.db + connection = db.connection qn = connection.ops.quote_name # self._select is a dictionary, and dictionaries' key order is @@ -612,7 +618,7 @@ columns = [f.column for f in fields] select = ['%s.%s' % (qn(self.model._meta.db_table), qn(c)) for c in columns] if extra_select: - select.extend(['(%s) AS %s' % (quote_only_if_word(s[1]), qn(s[0])) for s in extra_select]) + select.extend(['(%s) AS %s' % (quote_only_if_word(s[1], qn), qn(s[0])) for s in extra_select]) field_names.extend([f[0] for f in extra_select]) cursor = connection.cursor() @@ -638,6 +644,8 @@ from django.db.backends.util import typecast_timestamp from django.db.models.fields import DateTimeField + db = self.model._default_manager.db + connection = db.connection qn = connection.ops.quote_name self._order_by = () # Clear this because it'll mess things up otherwise. if self._field.null: @@ -783,7 +791,8 @@ return SortedDict(), [], [] return joins, where2, params -def get_where_clause(lookup_type, table_prefix, field_name, value, db_type): +def get_where_clause(opts, lookup_type, table_prefix, field_name, value, db_type): + connection = opts.get_default_manager().db.connection if table_prefix.endswith('.'): table_prefix = connection.ops.quote_name(table_prefix[:-1])+'.' field_name = connection.ops.quote_name(field_name) @@ -850,6 +859,7 @@ Helper function that recursively populates the select, tables and where (in place) for select_related queries. """ + connection = opts.get_default_manager().db.connection # If we've got a max_depth set and we've exceeded that depth, bail now. if max_depth and cur_depth > max_depth: @@ -953,13 +963,20 @@ return choices def lookup_inner(path, lookup_type, value, opts, table, column): - qn = connection.ops.quote_name joins, where, params = SortedDict(), [], [] current_opts = opts current_table = table current_column = column intermediate_table = None join_required = False + db = current_opts.get_default_manager().db + backend = db.backend + connection = db.connection + qn = connection.ops.quote_name + if hasattr(connection.ops, 'alias'): + al = connection.ops.alias + else: + al = lambda x,y: "%s__%s" % (x,y) name = path.pop(0) # Has the primary key been requested? If so, expand it out @@ -972,7 +989,7 @@ # Does the name belong to a defined many-to-many field? field = find_field(name, current_opts.many_to_many, False) if field: - new_table = current_table + '__' + name + new_table = al(current_table, name) new_opts = field.rel.to._meta new_column = new_opts.pk.column @@ -989,7 +1006,7 @@ # Does the name belong to a reverse defined many-to-many field? field = find_field(name, current_opts.get_all_related_many_to_many_objects(), True) if field: - new_table = current_table + '__' + name + new_table = al(current_table, name) new_opts = field.opts new_column = new_opts.pk.column @@ -1006,7 +1023,7 @@ # Does the name belong to a one-to-many field? field = find_field(name, current_opts.get_all_related_objects(), True) if field: - new_table = table + '__' + name + new_table = al(current_table, name) new_opts = field.opts new_column = field.field.column join_column = opts.pk.column @@ -1020,7 +1037,7 @@ field = find_field(name, current_opts.fields, False) if field: if field.rel: # One-to-One/Many-to-one field - new_table = current_table + '__' + name + new_table = al(current_table, name) new_opts = field.rel.to._meta new_column = new_opts.pk.column join_column = field.column @@ -1112,20 +1129,23 @@ column = field.column db_type = field.db_type() - where.append(get_where_clause(lookup_type, current_table + '.', column, value, db_type)) + where.append(get_where_clause(current_opts, lookup_type, current_table + '.', column, value, db_type)) params.extend(field.get_db_prep_lookup(lookup_type, value)) return joins, where, params def delete_objects(seen_objs): "Iterate through a list of seen classes, and remove any instances that are referred to" - qn = connection.ops.quote_name ordered_classes = seen_objs.keys() ordered_classes.reverse() - cursor = connection.cursor() - for cls in ordered_classes: + db = cls._default_manager.db + backend = db.backend + connection = db.connection + cursor = connection.cursor() + qn = connection.ops.quote_name + seen_objs[cls] = seen_objs[cls].items() seen_objs[cls].sort() @@ -1164,7 +1184,17 @@ pk_list[offset:offset+GET_ITERATOR_CHUNK_SIZE]) # Now delete the actual data + dirty_conns = [] for cls in ordered_classes: + + db = cls._default_manager.db + backend = db.backend + connection = db.connection + qn = connection.ops.quote_name + cursor = connection.cursor() + if connection not in dirty_conns: + dirty_conns.append(connection) + seen_objs[cls].reverse() pk_list = [pk for pk,instance in seen_objs[cls]] for offset in range(0, len(pk_list), GET_ITERATOR_CHUNK_SIZE): @@ -1183,4 +1213,4 @@ dispatcher.send(signal=signals.post_delete, sender=cls, instance=instance) setattr(instance, cls._meta.pk.attname, None) - transaction.commit_unless_managed() + transaction.commit_unless_managed(dirty_conns) === django/db/__init__.py ================================================================== --- django/db/__init__.py (/mirror/django/trunk) (revision 4246) +++ django/db/__init__.py (/local/django/multidb) (revision 4246) @@ -1,70 +1,380 @@ import os -from django.conf import settings +from django.conf import settings, UserSettingsHolder from django.core import signals from django.core.exceptions import ImproperlyConfigured from django.dispatch import dispatcher from django.utils.functional import curry +try: + # Only exists in Python 2.4+ + from threading import local +except ImportError: + # Import copy of _thread_local.py from Python 2.4 + from django.utils._threading_local import local + __all__ = ('backend', 'connection', 'DatabaseError', 'IntegrityError') + +# singleton to represent the default connection in connections +class dummy(object): + def __repr__(self): + return self.__str__() + def __str__(self): + return '' +_default = dummy() +del dummy + + +# storage for local default connection +_local = local() + if not settings.DATABASE_ENGINE: settings.DATABASE_ENGINE = 'dummy' + -try: - # Most of the time, the database backend will be one of the official - # backends that ships with Django, so look there first. - _import_path = 'django.db.backends.' - backend = __import__('%s%s.base' % (_import_path, settings.DATABASE_ENGINE), {}, {}, ['']) -except ImportError, e: - # If the import failed, we might be looking for a database backend - # distributed external to Django. So we'll try that next. - try: - _import_path = '' - backend = __import__('%s.base' % settings.DATABASE_ENGINE, {}, {}, ['']) - except ImportError, e_user: - # The database backend wasn't found. Display a helpful error message - # listing all possible (built-in) database backends. - backend_dir = os.path.join(__path__[0], 'backends') - available_backends = [f for f in os.listdir(backend_dir) if not f.startswith('_') and not f.startswith('.') and not f.endswith('.py') and not f.endswith('.pyc')] - available_backends.sort() - if settings.DATABASE_ENGINE not in available_backends: - raise ImproperlyConfigured, "%r isn't an available database backend. Available options are: %s" % \ - (settings.DATABASE_ENGINE, ", ".join(map(repr, available_backends))) - else: - raise # If there's some other error, this must be an error in Django itself. +def connect(settings, **kw): + """Connect to the database specified in settings. Returns a + ConnectionInfo on success, raises ImproperlyConfigured if the + settings don't specify a valid database connection. + """ + return ConnectionInfo(settings, **kw) -def _import_database_module(import_path='', module_name=''): - """Lazyily import a database module when requested.""" - return __import__('%s%s.%s' % (_import_path, settings.DATABASE_ENGINE, module_name), {}, {}, ['']) + +class ConnectionInfo(object): + """Encapsulates all information about a connection and the backend + to which it belongs. Provides methods for loading backend + creation, introspection, and shell modules, closing the + connection, and resetting the connection's query log. + """ + def __init__(self, settings=None, **kw): + super(ConnectionInfo, self).__init__(**kw) + if settings is None: + from django.conf import settings + if not settings.DATABASE_OPTIONS: + settings.DATABASE_OPTIONS = {} + self.settings = settings + self.backend = self.load_backend() + self.connection = self.backend.DatabaseWrapper(settings) + self.DatabaseError = self.backend.DatabaseError -# We don't want to import the introspect/creation modules unless -# someone asks for 'em, so lazily load them on demmand. -get_introspection_module = curry(_import_database_module, _import_path, 'introspection') -get_creation_module = curry(_import_database_module, _import_path, 'creation') + # Register an event that closes the database connection + # when a Django request is finished. + dispatcher.connect(self.close, signal=signals.request_finished) + + # Register an event that resets connection.queries + # when a Django request is started. + dispatcher.connect(self.reset_queries, signal=signals.request_started) -# We want runshell() to work the same way, but we have to treat it a -# little differently (since it just runs instead of returning a module like -# the above) and wrap the lazily-loaded runshell() method. -runshell = lambda: _import_database_module(_import_path, "client").runshell() + def __repr__(self): + return "Connection: %r (ENGINE=%s NAME=%s)" \ + % (self.connection, + self.settings.DATABASE_ENGINE, + self.settings.DATABASE_NAME) -# Convenient aliases for backend bits. -connection = backend.DatabaseWrapper(**settings.DATABASE_OPTIONS) -DatabaseError = backend.DatabaseError + def close(self): + """Close connection""" + self.connection.close() + + # We don't want to import the introspect/creation modules unless + # someone asks for 'em, so lazily load them on demmand. + def get_introspection_module(self): + return __import__('%s%s.%s' % (self._import_path, self.settings.DATABASE_ENGINE, 'introspection'), {}, {}, ['']) + + def get_creation_module(self): + return __import__('%s%s.%s' % (self._import_path, self.settings.DATABASE_ENGINE, 'creation'), {}, {}, ['']) + + def load_backend(self): + try: + # Most of the time, the database backend will be one of the official + # backends that ships with Django, so look there first. + self._import_path = 'django.db.backends.' + backend = __import__('%s%s.base' % (self._import_path, + self.settings.DATABASE_ENGINE), {}, {}, ['']) + except ImportError, e: + # If the import failed, we might be looking for a database backend + # distributed external to Django. So we'll try that next. + try: + self._import_path = '' + backend = __import__('%s.base' % self.settings.DATABASE_ENGINE, {}, {}, ['']) + except ImportError, e_user: + # The database backend wasn't found. Display a helpful error message + # listing all possible (built-in) database backends. + backend_dir = os.path.join(__path__[0], 'backends') + available_backends = [f for f in os.listdir(backend_dir) if not f.startswith('_') and not f.startswith('.') and not f.endswith('.py') and not f.endswith('.pyc')] + available_backends.sort() + if self.settings.DATABASE_ENGINE not in available_backends: + raise ImproperlyConfigured, "%r isn't an available database backend. Available options are: %s" % \ + (self.settings.DATABASE_ENGINE, ", ".join(map(repr, available_backends))) + else: + raise # If there's some other error, this must be an error in Django itself. + return backend + + # We want runshell() to work the same way, but we have to treat it a + # little differently (since it just runs instead of returning a module like + # the above) and wrap the lazily-loaded runshell() method. + def runshell(self): + return lambda: __import__('%s%s.%s' % (self._import_path, self.settings.DATABASE_ENGINE, 'client'), {}, {}, ['']).runshell() + + def reset_queries(self): + """Reset log of queries executed by connection""" + self.connection.queries = [] + +class LazyConnectionManager(object): + """Manages named connections lazily, instantiating them as + they are requested. + """ + def __init__(self): + self.local = local() + self.local.connections = {} + + # Reset connections on request finish, to make sure each request can + # load the correct connections for its settings + dispatcher.connect(self.reset, signal=signals.request_finished) + + def __iter__(self): + # Iterates only over *active* connections, not all possible + # connections + return iter(self.local.connections.keys()) + + def __getattr__(self, attr): + return getattr(self.local.connections, attr) + + def __getitem__(self, k): + try: + return self.local.connections[k] + except (AttributeError, KeyError): + return self.connect(k) + + def __setitem__(self, k, v): + try: + self.local.connections[k] = v + except AttributeError: + # First access in thread + self.local.connections = {k: v} + + def connect(self, name): + """Return the connection with this name in + settings.OTHER_DATABASES. Creates the connection if it doesn't yet + exist. Reconnects if it does. If the name requested is the default + connection (a singleton defined in django.db), then the default + connection is returned. + """ + try: + cnx = self.local.connections + except AttributeError: + cnx = self.local.connections = {} + if name in cnx: + cnx[name].close() + if name is _default: + cnx[name] = connect(settings) + return cnx[name] + try: + info = settings.OTHER_DATABASES[name] + except KeyError: + raise ImproperlyConfigured, \ + "No database connection '%s' has been configured" % name + except AttributeError: + raise ImproperlyConfigured, \ + "No OTHER_DATABASES in settings." + + # In settings it's a dict, but connect() needs an object: + # pass global settings so that the default connection settings + # can be defaults for the named connections. + database = UserSettingsHolder(settings) + for k, v in info.items(): + setattr(database, k, v) + cnx[name] = connect(database) + return cnx[name] + + def items(self): + # Iterates over *all possible* connections + items = [] + for key in self.keys(): + items.append((key, self[key])) + return items + + def keys(self): + # Iterates over *all possible* connections + keys = [_default] + try: + keys.extend(settings.OTHER_DATABASES.keys()) + except AttributeError: + pass + return keys + + def reset(self): + if not hasattr(self.local, 'connections'): + return + self.local.connections = {} + +def model_connection_name(klass): + """Get the connection name that a model is configured to use, with the + current settings. + """ + app = klass._meta.app_label + model = klass.__name__ + app_model = "%s.%s" % (app, model) + + # Quick exit if no OTHER_DATABASES defined + if (not hasattr(settings, 'OTHER_DATABASES') + or not settings.OTHER_DATABASES): + return _default + # Look in MODELS for the best match: app_label.Model. If that isn't + # found, take just app_label. If nothing is found, use the default + maybe = None + for name, db_def in settings.OTHER_DATABASES.items(): + if not 'MODELS' in db_def: + continue + mods = db_def['MODELS'] + # Can't get a better match than this + if app_model in mods: + return name + elif app in mods: + if maybe is not None: + raise ImproperlyConfigured, \ + "App %s appears in more than one OTHER_DATABASES " \ + "setting (%s and %s)" % (maybe, name) + maybe = name + if maybe: + return maybe + # No named connection for this model; use the default + return _default + + +class ConnectionInfoDescriptor(object): + """Descriptor used to access database connection information from a + manager or other connection holder. Keeps a thread-local cache of + connections per instance, and always returns the same connection for an + instance in particular thread during a particular request. + + Any object that includes a ``model`` attribute that holds a model class + can use this descriptor to manage connections. + """ + + def __init__(self): + self.local = local() + self.local.cnx = {} + dispatcher.connect(self.reset, signal=signals.request_finished) + + def __get__(self, instance, type=None): + if instance is None: + raise AttributeError, \ + "ConnectionInfo is accessible only through an instance" + try: + instance_connection = self.local.cnx.get(instance, None) + except AttributeError: + # First access in this thread + self.local.cnx = {} + instance_connection = None + if instance_connection is None: + instance_connection = self.get_connection(instance) + self.local.cnx[instance] = instance_connection + return instance_connection + + def __set__(self, instance, value): + try: + self.local.cnx[instance] = value + except AttributeError: + # First access in thread + self.local.cnx = {instance: value} + + def __delete__(self, instance): + try: + del self.local.cnx[instance] + except (AttributeError, KeyError): + # Not stored, no need to reset + pass + + def get_connection(self, instance): + return connections[model_connection_name(instance.model)] + + def reset(self): + if not hasattr(self.local, 'cnx'): + return + self.local.cnx = {} + +class LocalizingProxy: + """A lazy-initializing proxy. The proxied object is not + initialized until the first attempt to access it. This is used to + attach module-level properties to local storage. + """ + def __init__(self, name, storage, func, *arg, **kw): + self.__name = name + self.__storage = storage + self.__func = func + self.__arg = arg + self.__kw = kw + + # We need to clear out this thread's storage at the end of each + # request, in case new settings are loaded with the next + def reset(stor=storage, name=name): + if hasattr(stor, name): + delattr(stor, name) + dispatcher.connect(reset, signal=signals.request_finished) + + def __getattr__(self, attr): + # Private (__*) attributes are munged + if attr.startswith('_LocalizingProxy'): + return self.__dict__[attr] + try: + return getattr(getattr(self.__storage, self.__name), attr) + except AttributeError: + setattr(self.__storage, self.__name, self.__func(*self.__arg, + **self.__kw)) + return getattr(getattr(self.__storage, self.__name), attr) + + def __setattr__(self, attr, val): + # Private (__*) attributes are munged + if attr.startswith('_LocalizingProxy'): + self.__dict__[attr] = val + return + try: + stor = getattr(self.__storage, self.__name) + except AttributeError: + stor = self.__func(*self.__arg) + setattr(self.__storage, self.__name, stor) + setattr(stor, attr, val) + + +# Create a manager for named connections +connections = LazyConnectionManager() + +# Backwards compatibility: establish the default connection and set the +# default connection properties at module level, using the lazy proxy so that +# each thread may have a different default connection, if so configured +connection_info = LocalizingProxy('connection_info', _local, + lambda: connections[_default]) +connection = LocalizingProxy('connection', _local, + lambda: connections[_default].connection) +backend = LocalizingProxy('backend', _local, + lambda: connections[_default].backend) +DatabaseError = LocalizingProxy('DatabaseError', _local, + lambda: connections[_default].DatabaseError) +#================================BUG==Might need LocalizingProxy============== IntegrityError = backend.IntegrityError +get_introspection_module = LocalizingProxy( + 'get_introspection_module', _local, + lambda: connections[_default].get_introspection_module) +get_creation_module = LocalizingProxy( + 'get_creation_module', _local, + lambda: connections[_default].get_creation_module) +runshell = LocalizingProxy('runshell', _local, + lambda: connections[_default].runshell) -# Register an event that closes the database connection -# when a Django request is finished. -dispatcher.connect(connection.close, signal=signals.request_finished) -# Register an event that resets connection.queries -# when a Django request is started. -def reset_queries(): - connection.queries = [] -dispatcher.connect(reset_queries, signal=signals.request_started) - -# Register an event that rolls back the connection +# Register an event that rolls back all connections # when a Django request has an exception. def _rollback_on_exception(): from django.db import transaction transaction.rollback_unless_managed() -dispatcher.connect(_rollback_on_exception, signal=signals.got_request_exception) +dispatcher.connect(_rollback_on_exception, + signal=signals.got_request_exception) + +def reset_queries(): + connections[_default].reset_queries() + for c in connections: + try: + c.reset_queries() + except: + pass + === django/db/backends/ado_mssql/client.py ================================================================== --- django/db/backends/ado_mssql/client.py (/mirror/django/trunk) (revision 4246) +++ django/db/backends/ado_mssql/client.py (/local/django/multidb) (revision 4246) @@ -1,2 +1,2 @@ -def runshell(): +def runshell(settings): raise NotImplementedError === django/db/backends/mysql_old/base.py ================================================================== --- django/db/backends/mysql_old/base.py (/mirror/django/trunk) (revision 4246) +++ django/db/backends/mysql_old/base.py (/local/django/multidb) (revision 4246) @@ -148,8 +148,8 @@ 'iendswith': 'LIKE %s', } - def __init__(self, **kwargs): - super(DatabaseWrapper, self).__init__(**kwargs) + def __init__(self, settings): + super(DatabaseWrapper, self).__init__(settings) self.server_version = None def _valid_connection(self): === django/db/backends/postgresql/client.py ================================================================== --- django/db/backends/postgresql/client.py (/mirror/django/trunk) (revision 4246) +++ django/db/backends/postgresql/client.py (/local/django/multidb) (revision 4246) @@ -1,7 +1,6 @@ -from django.conf import settings import os -def runshell(): +def runshell(settings): args = ['psql'] if settings.DATABASE_USER: args += ["-U", settings.DATABASE_USER] === django/db/backends/sqlite3/base.py ================================================================== --- django/db/backends/sqlite3/base.py (/mirror/django/trunk) (revision 4246) +++ django/db/backends/sqlite3/base.py (/local/django/multidb) (revision 4246) @@ -115,7 +115,7 @@ return self.connection.cursor(factory=SQLiteCursorWrapper) def close(self): - from django.conf import settings + settings = self.settings # If database is in memory, closing the connection destroys the # database. To prevent accidental data loss, ignore close requests on # an in-memory db. === django/db/backends/sqlite3/client.py ================================================================== --- django/db/backends/sqlite3/client.py (/mirror/django/trunk) (revision 4246) +++ django/db/backends/sqlite3/client.py (/local/django/multidb) (revision 4246) @@ -1,6 +1,5 @@ -from django.conf import settings import os -def runshell(): +def runshell(settings): args = ['', settings.DATABASE_NAME] os.execvp('sqlite3', args) === django/db/backends/mysql/base.py ================================================================== --- django/db/backends/mysql/base.py (/mirror/django/trunk) (revision 4246) +++ django/db/backends/mysql/base.py (/local/django/multidb) (revision 4246) @@ -144,8 +144,8 @@ 'iendswith': 'LIKE %s', } - def __init__(self, **kwargs): - super(DatabaseWrapper, self).__init__(**kwargs) + def __init__(self, settings): + super(DatabaseWrapper, self).__init__(settings) self.server_version = None def _valid_connection(self): === django/db/backends/mysql/client.py ================================================================== --- django/db/backends/mysql/client.py (/mirror/django/trunk) (revision 4246) +++ django/db/backends/mysql/client.py (/local/django/multidb) (revision 4246) @@ -1,7 +1,6 @@ -from django.conf import settings import os -def runshell(): +def runshell(settings): args = [''] db = settings.DATABASE_OPTIONS.get('db', settings.DATABASE_NAME) user = settings.DATABASE_OPTIONS.get('user', settings.DATABASE_USER) === django/db/backends/oracle/base.py ================================================================== --- django/db/backends/oracle/base.py (/mirror/django/trunk) (revision 4246) +++ django/db/backends/oracle/base.py (/local/django/multidb) (revision 4246) @@ -412,6 +412,7 @@ return self.connection is not None def _cursor(self, settings): + settings = self.settings if not self._valid_connection(): if len(settings.DATABASE_HOST.strip()) == 0: settings.DATABASE_HOST = 'localhost' === django/db/backends/oracle/client.py ================================================================== --- django/db/backends/oracle/client.py (/mirror/django/trunk) (revision 4246) +++ django/db/backends/oracle/client.py (/local/django/multidb) (revision 4246) @@ -1,7 +1,6 @@ -from django.conf import settings import os -def runshell(): +def runshell(settings): dsn = settings.DATABASE_USER if settings.DATABASE_PASSWORD: dsn += "/%s" % settings.DATABASE_PASSWORD === django/db/backends/__init__.py ================================================================== --- django/db/backends/__init__.py (/mirror/django/trunk) (revision 4246) +++ django/db/backends/__init__.py (/local/django/multidb) (revision 4246) @@ -5,15 +5,16 @@ # Import copy of _thread_local.py from Python 2.4 from django.utils._threading_local import local -class BaseDatabaseWrapper(local): +class BaseDatabaseWrapper(object): """ Represents a database connection. """ ops = None - def __init__(self, **kwargs): + def __init__(self, settings): + self.settings = settings + self.options = settings.DATABASE_OPTIONS self.connection = None self.queries = [] - self.options = kwargs def _commit(self): if self.connection is not None: @@ -29,7 +30,7 @@ self.connection = None def cursor(self): - from django.conf import settings + settings = self.settings cursor = self._cursor(settings) if settings.DEBUG: return self.make_debug_cursor(cursor) === django/db/backends/dummy/base.py ================================================================== --- django/db/backends/dummy/base.py (/mirror/django/trunk) (revision 4246) +++ django/db/backends/dummy/base.py (/local/django/multidb) (revision 4246) @@ -33,8 +33,5 @@ _commit = complain _rollback = ignore - def __init__(self, **kwargs): - pass - def close(self): pass === django/db/transaction.py ================================================================== --- django/db/transaction.py (/mirror/django/trunk) (revision 4246) +++ django/db/transaction.py (/local/django/multidb) (revision 4246) @@ -16,7 +16,6 @@ import thread except ImportError: import dummy_thread as thread -from django.db import connection from django.conf import settings class TransactionManagementError(Exception): @@ -116,48 +115,67 @@ Puts the transaction manager into a manual state: managed transactions have to be committed explicitly by the user. If you switch off transaction management and there is a pending commit/rollback, the data will be - commited. + commited. Note that managed state applies across all connections. """ thread_ident = thread.get_ident() top = state.get(thread_ident, None) if top: top[-1] = flag if not flag and is_dirty(): - connection._commit() + for cx in all_connections(): + cx._commit() set_clean() else: raise TransactionManagementError("This code isn't under transaction management") -def commit_unless_managed(): +def commit_unless_managed(connections=None): """ Commits changes if the system is not in managed transaction mode. """ if not is_managed(): - connection._commit() + if connections is None: + connections = all_connections() + else: + connections = ensure_connections(connections) + for cx in connections: + cx._commit() else: set_dirty() -def rollback_unless_managed(): +def rollback_unless_managed(connections=None): """ Rolls back changes if the system is not in managed transaction mode. """ if not is_managed(): - connection._rollback() + if connections is None: + connections = all_connections() + for cx in connections: + cx._rollback() else: set_dirty() -def commit(): +def commit(connections=None): """ Does the commit itself and resets the dirty flag. """ - connection._commit() + if connections is None: + connections = all_connections() + else: + connections = ensure_connections(connections) + for cx in connections: + cx._commit() set_clean() -def rollback(): +def rollback(connections=None): """ This function does the rollback itself and resets the dirty flag. """ - connection._rollback() + if connections is None: + connections = all_connections() + else: + connections = ensure_connections(connections) + for cx in connections: + cx._rollback() set_clean() ############## @@ -179,7 +197,7 @@ leave_transaction_management() return _autocommit -def commit_on_success(func): +def commit_on_success(func, connections=None): """ This decorator activates commit on response. This way, if the view function runs successfully, a commit is made; if the viewfunc produces an exception, @@ -198,7 +216,7 @@ raise else: if is_dirty(): - commit() + commit(connections) return res finally: leave_transaction_management() @@ -220,3 +238,31 @@ leave_transaction_management() return _commit_manually + +########### +# HELPERS # +########### + +def all_connections(): + from django.db import connection, connections + return [connection] + [ c.connection + for c in connections.values() ] + +def ensure_connections(val): + from django.db import connections + conns = [] + if isinstance(val, basestring): + val = [val] + try: + iter(val) + except: + val = [val] + for cx in val: + if hasattr(cx, 'cursor'): + conns.append(cx) + elif hasattr(cx, 'connection'): + conns.append(cx.connection) + elif isinstance(cx, basestring): + conns.append(connections[cx].connection) + return conns + === django/conf/global_settings.py ================================================================== --- django/conf/global_settings.py (/mirror/django/trunk) (revision 4246) +++ django/conf/global_settings.py (/local/django/multidb) (revision 4246) @@ -118,6 +118,9 @@ DATABASE_PORT = '' # Set to empty string for default. Not used with sqlite3. DATABASE_OPTIONS = {} # Set to empty dictionary for default. +# Optional named database connections in addition to the default. +OTHER_DATABASES = {} + # Host for sending e-mail. EMAIL_HOST = 'localhost' @@ -343,6 +346,16 @@ # If None, a name of 'test_' + DATABASE_NAME will be assumed TEST_DATABASE_NAME = None +# Tuple of other test databases to create. Names in this tuple +# are suffixes that will be appended to TEST_DATABASE_NAME +TEST_DATABASES = [] + +# Models to assign to each test database. This must be a list of +# dicts, with each dict key being a name from TEST_DATABASES and value +# a list of models or app_labels that will use that database. +TEST_DATABASE_MODELS = [] + +############### FROM TRUNK ############################# # Strings used to set the character set and collation order for the test # database. These values are passed literally to the server, so they are # backend-dependent. If None, no special settings are sent (system defaults are === django/core/management/commands/sqlsequencereset.py ================================================================== --- django/core/management/commands/sqlsequencereset.py (/mirror/django/trunk) (revision 4246) +++ django/core/management/commands/sqlsequencereset.py (/local/django/multidb) (revision 4246) @@ -5,5 +5,8 @@ output_transaction = True def handle_app(self, app, **options): - from django.db import connection, models - return '\n'.join(connection.ops.sequence_reset_sql(self.style, models.get_models(app))) + from django.db import connection, models, model_connection_name + connection_output = [] + for model in models.get_models(app): + connection_output.extend(model._default_manager.db.connection.ops.sequence_reset_sql(self.style,[model])) + return '\n'.join(connection_output) === django/core/management/commands/flush.py ================================================================== --- django/core/management/commands/flush.py (/mirror/django/trunk) (revision 4246) +++ django/core/management/commands/flush.py (/local/django/multidb) (revision 4246) @@ -14,7 +14,7 @@ def handle_noargs(self, **options): from django.conf import settings - from django.db import connection, transaction, models + from django.db import connections, connection, transaction, models from django.dispatch import dispatcher from django.core.management.sql import sql_flush, emit_post_sync_signal @@ -32,7 +32,6 @@ pass sql_list = sql_flush(self.style, only_django=True) - if interactive: confirm = raw_input("""You have requested a flush of the database. This will IRREVERSIBLY DESTROY all data currently in the %r database, @@ -45,8 +44,8 @@ if confirm == 'yes': try: - cursor = connection.cursor() - for sql in sql_list: + for conn, sql in sql_list: + cursor = connections[conn].connection.cursor() cursor.execute(sql) except Exception, e: transaction.rollback_unless_managed() === django/core/management/commands/syncdb.py ================================================================== --- django/core/management/commands/syncdb.py (/mirror/django/trunk) (revision 4246) +++ django/core/management/commands/syncdb.py (/local/django/multidb) (revision 4246) @@ -19,7 +19,8 @@ help = "Create the database tables for all apps in INSTALLED_APPS whose tables haven't already been created." def handle_noargs(self, **options): - from django.db import connection, transaction, models + from django.db import model_connection_name + from django.db import transaction, models from django.conf import settings from django.core.management.sql import table_list, installed_models, sql_model_create, sql_for_pending_references, many_to_many_sql_for_model, custom_sql_for_model, sql_indexes_for_model, emit_post_sync_signal @@ -36,15 +37,9 @@ except ImportError: pass - cursor = connection.cursor() - - if connection.features.uses_case_insensitive_names: - table_name_converter = lambda x: x.upper() - else: - table_name_converter = lambda x: x # Get a list of all existing database tables, so we know what needs to # be added. - tables = [table_name_converter(name) for name in table_list()] + tables = table_list() # Get a list of already installed *models* so that references work right. seen_models = installed_models(tables) @@ -58,8 +53,15 @@ for model in model_list: # Create the model's database table, if it doesn't already exist. if verbosity >= 2: - print "Processing %s.%s model" % (app_name, model._meta.object_name) - if table_name_converter(model._meta.db_table) in tables: + print "Processing %s.%s model (%s))" % (app_name, model._meta.object_name, model_connection_name(model)) + connection = model._default_manager.db.connection + cursor = connection.cursor() + if connection.features.uses_case_insensitive_names: + table_name_converter = lambda x: x.upper() + else: + table_name_converter = lambda x: x + ctables = [table_name_converter(name) for name in tables] + if table_name_converter(model._meta.db_table) in ctables: continue sql, references = sql_model_create(model, self.style, seen_models) seen_models.add(model) @@ -80,6 +82,8 @@ model_list = models.get_models(app) for model in model_list: if model in created_models: + connection = model._default_manager.db.connection + cursor = connection.cursor() sql = many_to_many_sql_for_model(model, self.style) if sql: if verbosity >= 2: @@ -99,6 +103,8 @@ app_name = app.__name__.split('.')[-2] for model in models.get_models(app): if model in created_models: + connection = model._default_manager.db.connection + cursor = connection.cursor() custom_sql = custom_sql_for_model(model) if custom_sql: if verbosity >= 1: @@ -118,6 +124,8 @@ app_name = app.__name__.split('.')[-2] for model in models.get_models(app): if model in created_models: + connection = model._default_manager.db.connection + cursor = connection.cursor() index_sql = sql_indexes_for_model(model, self.style) if index_sql: if verbosity >= 1: @@ -135,3 +143,4 @@ # Install the 'initial_data' fixture, using format discovery from django.core.management import call_command call_command('loaddata', 'initial_data', verbosity=verbosity) + === django/core/management/validation.py ================================================================== --- django/core/management/validation.py (/mirror/django/trunk) (revision 4246) +++ django/core/management/validation.py (/local/django/multidb) (revision 4246) @@ -19,7 +19,7 @@ Returns number of errors. """ from django.conf import settings - from django.db import models, connection + from django.db import models, connections, model_connection_name from django.db.models.loading import get_app_errors from django.db.models.fields.related import RelatedObject @@ -30,6 +30,8 @@ for cls in models.get_models(app): opts = cls._meta + connection_name = model_connection_name(cls) + connection = connections[connection_name] # Do field-specific validation. for f in opts.fields: @@ -63,7 +65,7 @@ # Check that max_length <= 255 if using older MySQL versions. if settings.DATABASE_ENGINE == 'mysql': - db_version = connection.get_server_version() + db_version = connection.connection.get_server_version() if db_version < (5, 0, 3) and isinstance(f, (models.CharField, models.CommaSeparatedIntegerField, models.SlugField)) and f.max_length > 255: e.add(opts, '"%s": %s cannot have a "max_length" greater than 255 when you are using a version of MySQL prior to 5.0.3 (you are using %s).' % (f.name, f.__class__.__name__, '.'.join([str(n) for n in db_version[:3]]))) @@ -74,6 +76,11 @@ if f.rel.to not in models.get_models(): e.add(opts, "'%s' has relation with model %s, which has not been installed" % (f.name, rel_opts.object_name)) + #TODO: Fix this to allow relations that span databases by splitting querys up + rel_connection = model_connection_name(f.rel.to) + if rel_connection != connection_name: + e.add(opts, "'%s' is configured to use connection '%s' but has relation with '%s', which is configured to use connection '%s'" % (cls.__name__, connection_name, f.rel.to.__name__, rel_connection)) + rel_name = RelatedObject(f.rel.to, cls, f).get_accessor_name() rel_query_name = f.related_query_name() for r in rel_opts.fields: === django/core/management/sql.py ================================================================== --- django/core/management/sql.py (/mirror/django/trunk) (revision 4246) +++ django/core/management/sql.py (/local/django/multidb) (revision 4246) @@ -8,11 +8,21 @@ from sets import Set as set # Python 2.3 fallback def table_list(): - "Returns a list of all table names that exist in the database." - from django.db import connection, get_introspection_module - cursor = connection.cursor() - return get_introspection_module().get_table_list(cursor) + """Returns a list of all table names that exist in the database.""" + from django.db import connections + table_list = [] + for conn in connections: + table_list.extend(table_list_conn(conn)) + return table_list +def table_list_conn(conn): + from django.db import connections + try: + cursor = connections[conn].connection.cursor() + return connections[conn].get_introspection_module().get_table_list(cursor) + except: + return [] + def django_table_list(only_existing=False): """ Returns a list of all table names that have associated Django models and @@ -32,18 +42,34 @@ tables = [t for t in tables if t in existing] return tables +def django_table_list_conn(conn, only_existing=False): + from django.db import models + from django.db import model_connection_name + tables = [] + for app in models.get_apps(): + for model in models.get_models(app): + if model_connection_name(model)==conn: + tables.append(model._meta.db_table) + tables.extend([f.m2m_db_table() for f in model._meta.many_to_many]) + if only_existing: + existing = table_list_conn(conn) + tables = [t for t in tables if t in existing] + return tables + def installed_models(table_list): "Returns a set of all models that are installed, given a list of existing table names." from django.db import connection, models all_models = [] for app in models.get_apps(): for model in models.get_models(app): - all_models.append(model) - if connection.features.uses_case_insensitive_names: - converter = lambda x: x.upper() - else: - converter = lambda x: x - return set([m for m in all_models if converter(m._meta.db_table) in map(converter, table_list)]) + connection = model._default_manager.db.connection + if connection.features.uses_case_insensitive_names: + converter = lambda x: x.upper() + else: + converter = lambda x: x + if converter(model._meta.db_table) in map(converter, table_list): + all_models.append(converter(model)) + return set(all_models) def sequence_list(): "Returns a list of information about all DB sequences for all models in all apps." @@ -66,7 +92,7 @@ def sql_create(app, style): "Returns a list of the CREATE TABLE SQL statements for the given app." - from django.db import models + from django.db import models, model_connection_name from django.conf import settings if settings.DATABASE_ENGINE == 'dummy': @@ -81,22 +107,30 @@ # generate invalid SQL (leaving models out of known_models is harmless, so # we can be conservative). app_models = models.get_models(app) - final_output = [] + # final_output = [] known_models = set([model for model in installed_models(table_list()) if model not in app_models]) pending_references = {} + connection_output = {} + for model in app_models: + connection_name = model_connection_name(model) + f_output = connection_output.setdefault(connection_name, []) output, references = sql_model_create(model, style, known_models) - final_output.extend(output) + f_output.extend(output) for refto, refs in references.items(): pending_references.setdefault(refto, []).extend(refs) - final_output.extend(sql_for_pending_references(model, style, pending_references)) + f_output.extend(sql_for_pending_references(model, style, pending_references)) # Keep track of the fact that we've created the table for this model. known_models.add(model) + final_output = _collate(connection_output) + # Create the many-to-many join tables. for model in app_models: - final_output.extend(many_to_many_sql_for_model(model, style)) + connection_name = model_connection_name(model) + f_output = connection_output.setdefault(connection_name, []) + f_output.extend(many_to_many_sql_for_model(model,style)) # Handle references to tables that are from other apps # but don't exist physically. @@ -110,39 +144,45 @@ final_output.append('-- The following references should be added but depend on non-existent tables:') final_output.extend(alter_sql) + # convert Boundstatements into strings + final_output = map(str, final_output) + return final_output def sql_delete(app, style): "Returns a list of the DROP TABLE SQL statements for the given app." + from django.db import model_connection_name from django.db import connection, models, get_introspection_module from django.db.backends.util import truncate_name - introspection = get_introspection_module() - # This should work even if a connection isn't available - try: - cursor = connection.cursor() - except: - cursor = None - - # Figure out which tables already exist - if cursor: - table_names = introspection.get_table_list(cursor) - else: - table_names = [] - if connection.features.uses_case_insensitive_names: - table_name_converter = lambda x: x.upper() - else: - table_name_converter = lambda x: x - output = [] - qn = connection.ops.quote_name - + # Output DROP TABLE statements for standard application tables. to_delete = set() references_to_delete = {} + app_models = models.get_models(app) for model in app_models: + connection = model._default_manager.db.connection + introspection = model._default_manager.db.get_introspection_module() + # This should work even if a connection isn't available + try: + cursor = connection.cursor() + except: + cursor = None + # Figure out which tables already exist + if cursor: + table_names = introspection.get_table_list(cursor) + else: + table_names = [] + if connection.features.uses_case_insensitive_names: + table_name_converter = lambda x: x.upper() + else: + table_name_converter = lambda x: x + + qn = connection.ops.quote_name + if cursor and table_name_converter(model._meta.db_table) in table_names: # The table exists, so it needs to be dropped opts = model._meta @@ -153,6 +193,25 @@ to_delete.add(model) for model in app_models: + connection = model._default_manager.db.connection + introspection = model._default_manager.db.get_introspection_module() + # This should work even if a connection isn't available + try: + cursor = connection.cursor() + except: + cursor = None + # Figure out which tables already exist + if cursor: + table_names = introspection.get_table_list(cursor) + else: + table_names = [] + if connection.features.uses_case_insensitive_names: + table_name_converter = str.upper + else: + table_name_converter = lambda x: x + + qn = connection.ops.quote_name + if cursor and table_name_converter(model._meta.db_table) in table_names: # Drop the table now output.append('%s %s;' % (style.SQL_KEYWORD('DROP TABLE'), @@ -177,6 +236,25 @@ # Output DROP TABLE statements for many-to-many tables. for model in app_models: + connection = model._default_manager.db.connection + introspection = model._default_manager.db.get_introspection_module() + # This should work even if a connection isn't available + try: + cursor = connection.cursor() + except: + cursor = None + # Figure out which tables already exist + if cursor: + table_names = introspection.get_table_list(cursor) + else: + table_names = [] + if connection.features.uses_case_insensitive_names: + table_name_converter = str.upper + else: + table_name_converter = lambda x: x + + qn = connection.ops.quote_name + opts = model._meta for f in opts.many_to_many: if cursor and table_name_converter(f.m2m_db_table()) in table_names: @@ -190,9 +268,15 @@ # Close database connection explicitly, in case this output is being piped # directly into a database client, to avoid locking issues. - if cursor: - cursor.close() - connection.close() + for model in app_models: + connection = model._default_manager.db.connection + try: + cursor = connection.cursor() + except: + cursor = None + if cursor: + cursor.close() + connection.close() return output[::-1] # Reverse it, to deal with table dependencies. @@ -207,34 +291,42 @@ If only_django is True, then only table names that have associated Django models and are in INSTALLED_APPS will be included. """ - from django.db import connection - if only_django: - tables = django_table_list() - else: - tables = table_list() - statements = connection.ops.sql_flush(style, tables, sequence_list()) + from django.db import connections + statements = [] + for conn in connections: + if only_django: + tables = django_table_list_conn(conn) + else: + tables = table_list_conn(conn) + statements.extend((conn,f) for f in connections[conn].connection.ops.sql_flush(style, tables, sequence_list())) return statements def sql_custom(app): "Returns a list of the custom table modifying SQL statements for the given app." from django.db.models import get_models - output = [] + from django.db import model_connection_name + connection_output = {} app_models = get_models(app) app_dir = os.path.normpath(os.path.join(os.path.dirname(app.__file__), 'sql')) for model in app_models: - output.extend(custom_sql_for_model(model)) + connection_name = model_connection_name(model) + output = connection_output.setdefault(connection_name, []) + output.extend(map(str, custom_sql_for_model(model))) + return _collate(connection_output) - return output - def sql_indexes(app, style): "Returns a list of the CREATE INDEX SQL statements for all models in the given app." - from django.db import models - output = [] - for model in models.get_models(app): - output.extend(sql_indexes_for_model(model, style)) - return output + from django.db import model_connection_name + from django.db.models import get_models + connection_output = {} + for model in get_models(app): + opts = model._meta + connection_name = model_connection_name(model) + output = connection_output.setdefault(connection_name, []) + output.extend(map(str, sql_indexes_for_model(model, style))) + return _collate(connection_output) def sql_all(app, style): "Returns a list of CREATE TABLE SQL, initial-data inserts, and CREATE INDEX SQL for the given module." @@ -245,12 +337,13 @@ Returns the SQL required to create a single model, as a tuple of: (list_of_sql, pending_references_dict) """ - from django.db import connection, models + from django.db import models opts = model._meta final_output = [] table_output = [] pending_references = {} + connection = model._default_manager.db.connection qn = connection.ops.quote_name for f in opts.fields: col_type = f.db_type() @@ -314,10 +407,14 @@ """ Returns any ALTER TABLE statements to add constraints after the fact. """ - from django.db import connection from django.db.backends.util import truncate_name + connection = model._default_manager.db.connection qn = connection.ops.quote_name + if hasattr(connection.ops, 'max_constraint_length'): + mnl = connection.ops.max_constraint_length + else: + mnl = connection.ops.max_name_length final_output = [] if connection.features.supports_constraints: opts = model._meta @@ -332,18 +429,19 @@ # So we are careful with character usage here. r_name = '%s_refs_%s_%x' % (r_col, col, abs(hash((r_table, table)))) final_output.append(style.SQL_KEYWORD('ALTER TABLE') + ' %s ADD CONSTRAINT %s FOREIGN KEY (%s) REFERENCES %s (%s)%s;' % \ - (qn(r_table), truncate_name(r_name, connection.ops.max_name_length()), + (qn(r_table), truncate_name(r_name, mnl()), qn(r_col), qn(table), qn(col), connection.ops.deferrable_sql())) del pending_references[model] return final_output def many_to_many_sql_for_model(model, style): - from django.db import connection, models + from django.db import models from django.contrib.contenttypes import generic opts = model._meta final_output = [] + connection = model._default_manager.db.connection qn = connection.ops.quote_name for f in opts.many_to_many: if not isinstance(f.rel, generic.GenericRel): @@ -455,3 +553,25 @@ dispatcher.send(signal=models.signals.post_syncdb, sender=app, app=app, created_models=created_models, verbosity=verbosity, interactive=interactive) + +def _collate(connection_output, reverse=False): + from django.db import _default + final_output = [] + if len(connection_output.keys()) == 1: + # all for the default connection + for statements in connection_output.values(): + final_output.extend(statements) + if reverse: + final_output.reverse() + else: + for connection_name, statements in connection_output.items(): + if not statements: + continue + final_output.append(' -- The following statements are for connection: %s' % connection_name) + if reverse: + statements.reverse() + final_output.extend(statements) + final_output.append(' -- END statements for %s\n' % + connection_name) + return map(str, final_output) + === django/core/context_processors.py ================================================================== --- django/core/context_processors.py (/mirror/django/trunk) (revision 4246) +++ django/core/context_processors.py (/local/django/multidb) (revision 4246) @@ -33,8 +33,11 @@ context_extras = {} if settings.DEBUG and request.META.get('REMOTE_ADDR') in settings.INTERNAL_IPS: context_extras['debug'] = True - from django.db import connection - context_extras['sql_queries'] = connection.queries + from django.db import connections + sqlq = [] + for c in connections: + sqlq.extend(connections[c].connection.queries) + context_extras['sql_queries'] = sqlq return context_extras def i18n(request): === tests/modeltests/invalid_models/__init__.py ================================================================== --- tests/modeltests/invalid_models/__init__.py (/mirror/django/trunk) (revision 4246) +++ tests/modeltests/invalid_models/__init__.py (/local/django/multidb) (revision 4246) @@ -1 +0,0 @@ - === tests/modeltests/multiple_databases (new directory) ================================================================== === tests/modeltests/multiple_databases/__init__.py ================================================================== --- tests/modeltests/multiple_databases/__init__.py (/mirror/django/trunk) (revision 4246) +++ tests/modeltests/multiple_databases/__init__.py (/local/django/multidb) (revision 4246) @@ -0,0 +1 @@ +pass === tests/modeltests/multiple_databases/models.py ================================================================== --- tests/modeltests/multiple_databases/models.py (/mirror/django/trunk) (revision 4246) +++ tests/modeltests/multiple_databases/models.py (/local/django/multidb) (revision 4246) @@ -0,0 +1,221 @@ +""" +XXX. Using multiple database connections + +Django normally uses only a single database connection. However, +support is available for using any number of different, named +connections. Multiple database support is entirely optional and has +no impact on your application if you don't use it. + +Named connections are defined in your settings module. Create a +`OTHER_DATABASES` variable that is a dict, mapping connection names to their +particulars. The particulars are defined in a dict with the same keys +as the variable names as are used to define the default connection, with one +addition: MODELS. + +The MODELS item in an OTHER_DATABASES entry is a list of the apps and models +that will use that connection. + +Access to named connections is through `django.db.connections`, which +behaves like a dict: you access connections by name. Connections are +established lazily, when accessed. `django.db.connections[database]` +holds a `ConnectionInfo` instance, with the attributes: +`DatabaseError`, `backend`, `get_introspection_module`, +`get_creation_module`, and `runshell`. + +To access a model's connection, use its manager. The connection is available +at `model._default_manager.db.connection`. To find the backend or other +connection metadata, use `model._meta.db` to access the full ConnectionInfo +with connection metadata. +""" + +from django.db import models + +class Artist(models.Model): + name = models.CharField(maxlength=100) + alive = models.BooleanField(default=True) + + def __str__(self): + return self.name + + +class Opus(models.Model): + artist = models.ForeignKey(Artist) + name = models.CharField(maxlength=100) + year = models.IntegerField() + + def __str__(self): + return "%s (%s)" % (self.name, self.year) + + +class Widget(models.Model): + code = models.CharField(maxlength=10, unique=True) + weight = models.IntegerField() + + def __str__(self): + return self.code + + +class DooHickey(models.Model): + name = models.CharField(maxlength=50) + widgets = models.ManyToManyField(Widget, related_name='doohickeys') + + def __str__(self): + return self.name + + +class Vehicle(models.Model): + make = models.CharField(maxlength=20) + model = models.CharField(maxlength=20) + year = models.IntegerField() + + def __str__(self): + return "%d %s %s" % (self.year, self.make, self.model) + + +__test__ = {'API_TESTS': """ + +# See what connections are defined. django.db.connections acts like a dict. +>>> from django.db import connection, connections, _default, model_connection_name +>>> from django.conf import settings + +# Connections are referenced by name +>>> connections['_a'] +Connection: ... +>>> connections['_b'] +Connection: ... + +# Let's see what connections are available. The default connection is always +# included in connections as well, and may be accessed as connections[_default]. + +>>> connection_names = connections.keys() +>>> connection_names.sort() +>>> connection_names +[, '_a', '_b'] + +# Invalid connection names raise ImproperlyConfigured + +>>> connections['bad'] +Traceback (most recent call last): + ... +ImproperlyConfigured: No database connection 'bad' has been configured + +# The model_connection_name() function will tell you the name of the +# connection that a model is configured to use. + +>>> model_connection_name(Artist) +'_a' +>>> model_connection_name(Widget) +'_b' +>>> model_connection_name(Vehicle) is _default +True +>>> a = Artist(name="Paul Klee", alive=False) +>>> a.save() +>>> w = Widget(code='100x2r', weight=1000) +>>> w.save() +>>> v = Vehicle(make='Chevy', model='Camaro', year='1966') +>>> v.save() +>>> artists = Artist.objects.all() +>>> list(artists) +[] + +# Models can access their connections through the db property of their +# default manager. + +>>> paul = _[0] +>>> Artist.objects.db +Connection: ... (ENGINE=... NAME=...) +>>> paul._default_manager.db +Connection: ... (ENGINE=... NAME=...) + +# When transactions are not managed, model save will commit only +# for the model's connection. + +>>> from django.db import transaction +>>> transaction.enter_transaction_management() +>>> transaction.managed(False) +>>> a = Artist(name="Joan Miro", alive=False) +>>> w = Widget(code="99rbln", weight=1) +>>> a.save() + +# Only connection '_a' is committed, so if we rollback +# all connections we'll forget the new Widget. + +>>> transaction.rollback() +>>> list(Artist.objects.all()) +[, ] +>>> list(Widget.objects.all()) +[] + +# Managed transaction state applies across all connections. + +>>> transaction.managed(True) + +# When managed, just as when using a single connection, updates are +# not committed until a commit is issued. + +>>> a = Artist(name="Pablo Picasso", alive=False) +>>> a.save() +>>> w = Widget(code="99rbln", weight=1) +>>> w.save() +>>> v = Vehicle(make='Pontiac', model='Fiero', year='1987') +>>> v.save() + +# The connections argument may be passed to commit, rollback, and the +# commit_on_success decorator as a keyword argument, as the first (for +# commit and rollback) or second (for the decorator) positional +# argument. It may be passed as a ConnectionInfo object, a connection +# (DatabaseWrapper) object, a connection name, or a list or dict of +# ConnectionInfo objects, connection objects, or connection names. If a +# dict is passed, the keys are ignored and the values used as the list +# of connections to commit, rollback, etc. + +>>> transaction.commit(connections['_b']) +>>> transaction.commit('_b') +>>> transaction.commit(connections='_b') +>>> transaction.commit(connections=['_b']) +>>> transaction.commit(['_a', '_b']) +>>> transaction.commit(connections) + +# When the connections argument is omitted entirely, the transaction +# command applies to all connections. Here we have committed +# connections 'django_test_db_a' and 'django_test_db_b', but not the +# default connection, so the new vehicle is lost on rollback. + +>>> transaction.rollback() +>>> list(Artist.objects.all()) +[, , ] +>>> list(Widget.objects.all()) +[, ] +>>> list(Vehicle.objects.all()) +[] +>>> transaction.rollback() +>>> transaction.managed(False) +>>> transaction.leave_transaction_management() + +# Of course, relations and all other normal database operations work +# with models that use named connections just the same as with models +# that use the default connection. The only caveat is that you can't +# use a relation between two models that are stored in different +# databases. Note that that doesn't mean that two models using +# different connection *names* can't be related; only that in the the +# context in which they are used, if you use the relation, the +# connections named by the two models must resolve to the same +# database. + +>>> a = Artist.objects.get(name="Paul Klee") +>>> list(a.opus_set.all()) +[] +>>> a.opus_set.create(name="Magic Garden", year="1926") + +>>> list(a.opus_set.all()) +[] +>>> d = DooHickey(name='Thing') +>>> d.save() +>>> d.widgets.create(code='d101', weight=92) + +>>> list(d.widgets.all()) +[] +>>> w = Widget.objects.get(code='d101') +>>> list(w.doohickeys.all()) +[] +"""} Property changes on: tests/modeltests/multiple_databases ___________________________________________________________________ Name: svn:ignore +*.pyc + === tests/regressiontests/manager_db (new directory) ================================================================== === tests/regressiontests/manager_db/__init__.py ================================================================== === tests/regressiontests/manager_db/tests.py ================================================================== --- tests/regressiontests/manager_db/tests.py (/mirror/django/trunk) (revision 4246) +++ tests/regressiontests/manager_db/tests.py (/local/django/multidb) (revision 4246) @@ -0,0 +1,17 @@ +import unittest +from regressiontests.manager_db.models import Insect + +class TestManagerDBAccess(unittest.TestCase): + + def test_db_property(self): + m = Insect.objects + db = Insect.objects.db + assert db + assert db.connection + assert db.connection.cursor + assert db.backend + assert db.connection.ops.quote_name + assert db.get_creation_module + +if __name__ == '__main__': + unittest.main() === tests/regressiontests/manager_db/models.py ================================================================== --- tests/regressiontests/manager_db/models.py (/mirror/django/trunk) (revision 4246) +++ tests/regressiontests/manager_db/models.py (/local/django/multidb) (revision 4246) @@ -0,0 +1,5 @@ +from django.db import models + +class Insect(models.Model): + common_name = models.CharField(maxlength=64) + latin_name = models.CharField(maxlength=128) Property changes on: tests/regressiontests/manager_db ___________________________________________________________________ Name: svn:ignore +*.pyc + === tests/regressiontests/thread_isolation (new directory) ================================================================== === tests/regressiontests/thread_isolation/__init__.py ================================================================== === tests/regressiontests/thread_isolation/tests.py ================================================================== --- tests/regressiontests/thread_isolation/tests.py (/mirror/django/trunk) (revision 4246) +++ tests/regressiontests/thread_isolation/tests.py (/local/django/multidb) (revision 4246) @@ -0,0 +1,251 @@ +# tests that db settings can be different in different threads +# +# +# What's going on here: +# +# Simulating multiple web requests in a threaded environment, one in +# which settings are different for each request. So we replace +# django.conf.settings with a thread local, with different +# configurations in each thread, and then fire off three +# simultaneous requests (using a condition to sync them up), and +# test that each thread sees its own settings and the models in each +# thread attempt to connect to the correct database as per their +# settings. +# + + +import copy +import os +import sys +import threading +import unittest +from thread import get_ident + +from django.conf import settings, UserSettingsHolder +from django.core.handlers.wsgi import WSGIHandler +from django.db import model_connection_name, _default, connection, connections +from regressiontests.request_isolation.tests import MockHandler +from regressiontests.thread_isolation.models import * + +try: + # Only exists in Python 2.4+ + from threading import local +except ImportError: + # Import copy of _thread_local.py from Python 2.4 + from django.utils._threading_local import local + +# helpers +EV = threading.Event() + +class LocalSettings: + """Settings holder that allows thread-local overrides of defaults. + """ + def __init__(self, defaults): + self._defaults = defaults + self._local = local() + + def __getattr__(self, attr): + if attr in ('_defaults', '_local'): + return self.__dict__[attr] + _local = self.__dict__['_local'] + _defaults = self.__dict__['_defaults'] + debug("LS get %s (%s)", attr, hasattr(_local, attr)) + if not hasattr(_local, attr): + # Make sure everything we return is the local version; this + # avoids sets to deep datastructures overwriting the defaults + setattr(_local, attr, copy.deepcopy(getattr(_defaults, attr))) + return getattr(_local, attr) + + def __setattr__(self, attr, val): + if attr in ('_defaults', '_local'): + self.__dict__[attr] = val + else: + debug("LS set local %s = %s", attr, val) + setattr(self.__dict__['_local'], attr, val) + +def thread_two(func, *arg): + def start(): + # from django.conf import settings + settings.OTHER_DATABASES['_b']['MODELS'] = [] + + debug("t2 ODB: %s", settings.OTHER_DATABASES) + debug("t2 waiting") + EV.wait(2.0) + func(*arg) + debug("t2 complete") + t2 = threading.Thread(target=start) + t2.start() + return t2 + +def thread_three(func, *arg): + def start(): + # from django.conf import settings + settings.OTHER_DATABASES['_b']['MODELS'] = ['ti.MY'] + settings.OTHER_DATABASES['_b'], \ + settings.OTHER_DATABASES['_a'] = \ + settings.OTHER_DATABASES['_a'], \ + settings.OTHER_DATABASES['_b'] + + settings.DATABASE_NAME = \ + settings.OTHER_DATABASES['_a']['DATABASE_NAME'] + + debug("t3 ODB: %s", settings.OTHER_DATABASES) + debug("3 %s: start: default: %s", get_ident(), settings.DATABASE_NAME) + debug("3 %s: start: conn: %s", get_ident(), + connection.settings.DATABASE_NAME) + + debug("t3 waiting") + EV.wait(2.0) + func(*arg) + debug("t3 complete") + t3 = threading.Thread(target=start) + t3.start() + return t3 + +def debug(*arg): + pass +# msg, arg = arg[0], arg[1:] +# print msg % arg + +def start_response(code, headers): + debug("start response: %s %s", code, headers) + pass + +class TestThreadIsolation(unittest.TestCase): + # event used to synchronize threads so we can be sure they are running + # together + lock = threading.RLock() + errors = [] + + def setUp(self): + debug("setup") + self.settings = settings._target + settings._target = UserSettingsHolder(copy.deepcopy(settings._target)) + settings.OTHER_DATABASES['_a']['MODELS'] = ['ti.MX'] + settings.OTHER_DATABASES['_b']['MODELS'] = ['ti.MY'] + + # normal settings holders aren't thread-safe, so we need to substitute + # one that is (and so allows per-thread settings) + holder = settings._target + settings._target = LocalSettings(holder) + + def teardown(self): + debug("teardown") + settings._target = self.settings + + def add_thread_error(self, err): + self.lock.acquire() + try: + self.errors.append(err) + finally: + self.lock.release() + + def thread_errors(self): + self.lock.acquire() + try: + return self.errors[:] + finally: + self.lock.release() + + def request_one(self, request): + """Start out with settings as originally configured""" + from django.conf import settings + debug("request_one: %s", settings.OTHER_DATABASES) + + self.assertEqual(model_connection_name(MQ), _default) + self.assertEqual(model_connection_name(MX), '_a') + self.assertEqual( + MX._default_manager.db.connection.settings.DATABASE_NAME, + settings.OTHER_DATABASES['_a']['DATABASE_NAME']) + self.assertEqual(model_connection_name(MY), '_b') + self.assertEqual( + MY._default_manager.db.connection.settings.DATABASE_NAME, + settings.OTHER_DATABASES['_b']['DATABASE_NAME']) + self.assert_(MQ._default_manager.db.connection is + connections[_default].connection) + self.assertEqual( + MQ._default_manager.db.connection.settings.DATABASE_NAME, + settings.DATABASE_NAME) + self.assertEqual(connection.settings.DATABASE_NAME, + settings.DATABASE_NAME) + + def request_two(self, request): + """Between the first and second requests, settings change to assign + model MY to a different connection + """ + # from django.conf import settings + debug("request_two: %s", settings.OTHER_DATABASES) + + try: + self.assertEqual(model_connection_name(MQ), _default) + self.assertEqual(model_connection_name(MX), '_a') + self.assertEqual( + MX._default_manager.db.connection.settings.DATABASE_NAME, + settings.OTHER_DATABASES['_a']['DATABASE_NAME']) + self.assertEqual(model_connection_name(MY), _default) + self.assertEqual( + MY._default_manager.db.connection.settings.DATABASE_NAME, + settings.DATABASE_NAME) + self.assert_(MQ._default_manager.db.connection is + connections[_default].connection) + self.assertEqual( + MQ._default_manager.db.connection.settings.DATABASE_NAME, + settings.DATABASE_NAME) + self.assertEqual(connection.settings.DATABASE_NAME, + settings.DATABASE_NAME) + except: + self.add_thread_error(sys.exc_info()) + + def request_three(self, request): + """Between the 2nd and 3rd requests, the settings at the names in + OTHER_DATABASES have changed. + """ + # from django.conf import settings + debug("3 %s: %s", get_ident(), settings.OTHER_DATABASES) + debug("3 %s: default: %s", get_ident(), settings.DATABASE_NAME) + debug("3 %s: conn: %s", get_ident(), + connection.settings.DATABASE_NAME) + try: + self.assertEqual(model_connection_name(MQ), _default) + self.assertEqual(model_connection_name(MX), '_b') + self.assertEqual( + MX._default_manager.db.connection.settings.DATABASE_NAME, + settings.OTHER_DATABASES['_b']['DATABASE_NAME']) + self.assertEqual(model_connection_name(MY), '_a') + self.assertEqual( + MY._default_manager.db.connection.settings.DATABASE_NAME, + settings.OTHER_DATABASES['_a']['DATABASE_NAME']) + self.assert_(MQ._default_manager.db.connection is + connections[_default].connection) + self.assertEqual( + connection.settings.DATABASE_NAME, + settings.OTHER_DATABASES['_a']['DATABASE_NAME']) + except: + self.add_thread_error(sys.exc_info()) + + def test_thread_isolation(self): + + debug("running tests") + + env = os.environ.copy() + env['PATH_INFO'] = '/' + env['REQUEST_METHOD'] = 'GET' + + t2 = thread_two(MockHandler(self.request_two), env, start_response) + t3 = thread_three(MockHandler(self.request_three), env, start_response) + + try: + EV.set() + MockHandler(self.request_one)(env, start_response) + finally: + t2.join() + t3.join() + err = self.thread_errors() + if err: + import traceback + for e in err: + traceback.print_exception(*e) + raise AssertionError("%s thread%s failed" % + (len(err), len(err) > 1 and 's' or + '')) + === tests/regressiontests/thread_isolation/models.py ================================================================== --- tests/regressiontests/thread_isolation/models.py (/mirror/django/trunk) (revision 4246) +++ tests/regressiontests/thread_isolation/models.py (/local/django/multidb) (revision 4246) @@ -0,0 +1,19 @@ +from django.db import models + +# models +class MQ(models.Model): + val = models.CharField(maxlength=10) + class Meta: + app_label = 'ti' + + +class MX(models.Model): + val = models.CharField(maxlength=10) + class Meta: + app_label = 'ti' + + +class MY(models.Model): + val = models.CharField(maxlength=10) + class Meta: + app_label = 'ti' Property changes on: tests/regressiontests/thread_isolation ___________________________________________________________________ Name: svn:ignore +*.pyc + Property changes on: tests/regressiontests/initial_sql_regress/sql ___________________________________________________________________ Name: svn:ignore +*.pyc + === tests/regressiontests/request_isolation (new directory) ================================================================== === tests/regressiontests/request_isolation/__init__.py ================================================================== === tests/regressiontests/request_isolation/tests.py ================================================================== --- tests/regressiontests/request_isolation/tests.py (/mirror/django/trunk) (revision 4246) +++ tests/regressiontests/request_isolation/tests.py (/local/django/multidb) (revision 4246) @@ -0,0 +1,100 @@ +# tests that db settings can change between requests +import copy +import os +import unittest +from django.conf import settings, UserSettingsHolder +from django.core.handlers.wsgi import WSGIHandler +from django.db import models, model_connection_name, _default, connection +from django.http import HttpResponse +from regressiontests.request_isolation.models import * + + +# helpers +class MockHandler(WSGIHandler): + + def __init__(self, test): + self.test = test + super(MockHandler, self).__init__() + + def get_response(self, request): + # debug("mock handler answering %s, %s", path, request) + return HttpResponse(self.test(request)) + + +def debug(*arg): + pass + # msg, arg = arg[0], arg[1:] + # print msg % arg + + +def start_response(code, headers): + debug("start response: %s %s", code, headers) + pass + +# tests +class TestRequestIsolation(unittest.TestCase): + + def setUp(self): + debug("setup") + self.settings = settings._target + settings._target = UserSettingsHolder(copy.deepcopy(settings._target)) + settings.OTHER_DATABASES['_a']['MODELS'] = ['ri.MX'] + settings.OTHER_DATABASES['_b']['MODELS'] = ['ri.MY'] + + def tearDown(self): + debug("teardown") + settings._target = self.settings + + def testRequestIsolation(self): + env = os.environ.copy() + env['PATH_INFO'] = '/' + env['REQUEST_METHOD'] = 'GET' + + def request_one(request): + """Start out with settings as originally configured""" + self.assertEqual(model_connection_name(MX), '_a') + self.assertEqual( + MX._default_manager.db.connection.settings.DATABASE_NAME, + settings.OTHER_DATABASES['_a']['DATABASE_NAME']) + self.assertEqual(model_connection_name(MY), '_b') + self.assertEqual( + MY._default_manager.db.connection.settings.DATABASE_NAME, + settings.OTHER_DATABASES['_b']['DATABASE_NAME']) + + def request_two(request): + """Between the first and second requests, settings change to assign + model MY to a different connection + """ + self.assertEqual(model_connection_name(MX), '_a') + self.assertEqual( + MX._default_manager.db.connection.settings.DATABASE_NAME, + settings.OTHER_DATABASES['_a']['DATABASE_NAME']) + self.assertEqual(model_connection_name(MY), _default) + self.assertEqual( + MY._default_manager.db.connection.settings.DATABASE_NAME, + settings.DATABASE_NAME) + + def request_three(request): + """Between the 2nd and 3rd requests, the settings at the names in + OTHER_DATABASES have changed. + """ + self.assertEqual(model_connection_name(MX), '_b') + self.assertEqual( + MX._default_manager.db.connection.settings.DATABASE_NAME, + settings.OTHER_DATABASES['_b']['DATABASE_NAME']) + self.assertEqual(model_connection_name(MY), '_a') + self.assertEqual( + MY._default_manager.db.connection.settings.DATABASE_NAME, + settings.OTHER_DATABASES['_a']['DATABASE_NAME']) + + MockHandler(request_one)(env, start_response) + + settings.OTHER_DATABASES['_b']['MODELS'] = [] + MockHandler(request_two)(env, start_response) + + settings.OTHER_DATABASES['_b']['MODELS'] = ['ri.MY'] + settings.OTHER_DATABASES['_b'], \ + settings.OTHER_DATABASES['_a'] = \ + settings.OTHER_DATABASES['_a'], \ + settings.OTHER_DATABASES['_b'] + MockHandler(request_three)(env, start_response) === tests/regressiontests/request_isolation/models.py ================================================================== --- tests/regressiontests/request_isolation/models.py (/mirror/django/trunk) (revision 4246) +++ tests/regressiontests/request_isolation/models.py (/local/django/multidb) (revision 4246) @@ -0,0 +1,13 @@ +from django.db import models + +# models +class MX(models.Model): + val = models.CharField(maxlength=10) + class Meta: + app_label = 'ri' + + +class MY(models.Model): + val = models.CharField(maxlength=10) + class Meta: + app_label = 'ri' Property changes on: tests/regressiontests/request_isolation ___________________________________________________________________ Name: svn:ignore +*.pyc + === tests/runtests.py ================================================================== --- tests/runtests.py (/mirror/django/trunk) (revision 4246) +++ tests/runtests.py (/local/django/multidb) (revision 4246) @@ -18,6 +18,14 @@ TEST_TEMPLATE_DIR = 'templates' CONTRIB_DIR = os.path.dirname(contrib.__file__) +TEST_OTHER_DATABASES = { + '_a': { 'DATABASE_NAME': 'django_test_a', + 'MODELS': [ 'multiple_databases.Artist', + 'multiple_databases.Opus' ]}, + '_b': { 'DATABASE_NAME': 'django_test_b', + 'MODELS': [ 'multiple_databases.Widget', + 'multiple_databases.DooHickey' ]} +} MODEL_TEST_DIR = os.path.join(os.path.dirname(__file__), MODEL_TESTS_DIR_NAME) REGRESSION_TEST_DIR = os.path.join(os.path.dirname(__file__), REGRESSION_TESTS_DIR_NAME) @@ -97,6 +105,7 @@ # Redirect some settings for the duration of these tests. settings.INSTALLED_APPS = ALWAYS_INSTALLED_APPS + settings.TEST_OTHER_DATABASES = TEST_OTHER_DATABASES settings.ROOT_URLCONF = 'urls' settings.TEMPLATE_DIRS = (os.path.join(os.path.dirname(__file__), TEST_TEMPLATE_DIR),) settings.USE_I18N = True Property changes on: tests/templates ___________________________________________________________________ Name: svn:ignore +*.pyc + === docs/settings.txt ================================================================== --- docs/settings.txt (/mirror/django/trunk) (revision 4246) +++ docs/settings.txt (/local/django/multidb) (revision 4246) @@ -667,6 +667,13 @@ See `allowed date format strings`_. See also ``DATE_FORMAT``, ``DATETIME_FORMAT``, ``TIME_FORMAT`` and ``YEAR_MONTH_FORMAT``. +OTHER_DATABASES +--------------- + +Default: ``{}`` + +Other database connections to use in addition to the default connection. See the `multiple database support docs`_. + PREPEND_WWW ----------- === docs/multiple_database_support.txt ================================================================== --- docs/multiple_database_support.txt (/mirror/django/trunk) (revision 4246) +++ docs/multiple_database_support.txt (/local/django/multidb) (revision 4246) @@ -0,0 +1,163 @@ +======================== +Using Multiple Databases +======================== + +Standard Django practice is to use a single database connection for +all models in all applications. However, Django supports configuring +and using multiple database connections on a per-application, per-model +or an ad-hoc basis. Using multiple database connections is optional. + +Configuring other database connections +====================================== + +Django's default database connection is configured via the settings +DATABASE_ENGINE, DATABASE_NAME, DATABASE_USER, DATABASE_PASSWORD, +DATABASE_HOST, and DATABASE_PORT. Other connections are configured via +the OTHER_DATABASES setting. Define OTHER_DATABASES as a dict, with a +name for each connection as the key and a dict of settings as the +value. In each OTHER_DATABASES entry (called a "named connection"), +the keys are the same as the DATABASE_ENGINE, etc, settings used to +configure the default connection. All keys are optional; any that are +missing in a named connection's settings will inherit their values +from the default connection. + +Here's an example:: + + DATABASE_ENGINE = 'postgresql' + DATABASE_NAME = 'django_apps' + DATABASE_USER = 'default_user' + DATABASE_PASSWORD = 'xxx' + + OTHER_DATABASES = { + 'local': { 'DATABASE_ENGINE': 'sqlite3', + 'DATABASE_NAME': '/tmp/cache.db' }, + 'public': { 'DATABASE_HOST': 'public', + 'DATABASE_USER': 'public_user', + 'DATABASE_PASSWORD': 'xxx' } + 'private': { 'DATABASE_HOST': 'private', + 'DATABASE_USER': 'private_user', + 'DATABASE_PASSWORD': 'xxx' } + } + +In addition to the DATABASE_* settings, each named connection in +OTHER_DATABASES may optionally include a MODELS setting. This should +be a list of app or app.model names, and is used to configure which +models should use this connection instead of the default connection. + +Here's the example above, with ``MODELS``:: + + OTHER_DATABASES = { + 'local': { 'DATABASE_ENGINE': 'sqlite3', + 'DATABASE_NAME': '/tmp/cache.db', + # A model name: only the model ContentItem + # with the app_label myapp will use this connection + 'MODELS': ['myapp.ContentItem'] }, + 'public': { 'DATABASE_HOST': 'public', + 'DATABASE_USER': 'public_user', + 'DATABASE_PASSWORD': 'xxx', + # Two models in myapp will use the connection + # named 'public', as will ALL models in + # django.contribe.comments + 'MODELS': ['myapp.Blog','myapp.Article', + 'django.contrib.comments' ] } + # No models or apps are configured to use the private db + 'private': { 'DATABASE_HOST': 'private', + 'DATABASE_USER': 'private_user', + 'DATABASE_PASSWORD': 'xxx' } + } + +Accessing a model's connection +============================== + +Each manager has a ``db`` attribute that can be used to access the model's +connection. Access the ``db`` attribute of a model's manager to obtain the +model's currently configured connection. + +Example:: + + from django.db import models + + class Blog(models.Model) + name = models.CharField(maxlength=50) + + class Article(models.Model) + blog = models.ForeignKey(Blog) + title = models.CharField(maxlength=100) + slug = models.SlugField() + summary = models.CharField(maxlength=500) + body = models.TextField() + + class ContentItem(models.Model) + slug = models.SlugField() + mimetype = models.CharField(maxlength=50) + file = models.FileField() + + # Get a ConnectionInfo instance that describes the connection + article_db = Article.objects.db + + # Get a connection and a cursor + connection = article_db.connection + cursor = connection.cursor() + + # Get the ``quote_name`` function from the backend + qn = article_db.backend.quote_name + +Ordinarily you won't have to access a model's connection directly; +just use the model and manager normally and they will use the +connection configured for the model. + +ConnectionInfo objects +====================== + +FIXME Describe the ConnectionInfo object and each of its attributes. + + +Accessing connections by name +============================= + +Access named connections directly through +``django.db.connections``. Each entry in ``django.db.connections`` is +a ``ConnectionInfo`` instance bound to the settings configured in the +OTHER_DATABASES entry under the same key. + +Example:: + + from django.db import connections + + private_db = connections['private'] + cursor = private_db.connection.cursor() + + +Using transactions with other database connections +================================================== + +Transaction managed state applies across all connections +commit/rollback apply to all connections by default +but you can specify individual connections or lists or dicts of connections + + +Changing model connections on the fly +===================================== + +Here's an example of primitive mirroring:: + + # Read all articles from the private db + # Note that we pull the articles into a list; this is necessary + # because query sets are lazy. If we were to change the model's + # connection without copying the articles into a local list, we'd + # wind up reading from public instead of private. + + Article.objects.db = connections['private'] + all_articles = list(Article.objects.all()) + + # Save each article in the public db + Article.objects.db = connections['public'] + for article in all_articles: + article.save() + +Thread and request isolation +============================ + +connections close after each request +connection settings are thread-local + Property changes on: ___________________________________________________________________ Name: svk:merge -bcc190cf-cafb-0310-a4f2-bffc1f526a37:/django/trunk:1054 +bbbf0183-8649-de40-ae96-8c00a8d06f91:/local/django/db2_9:3912 +bcc190cf-cafb-0310-a4f2-bffc1f526a37:/django/trunk:6453