Index: django/db/models/sql/compiler.py =================================================================== --- django/db/models/sql/compiler.py (revision 15147) +++ django/db/models/sql/compiler.py (working copy) @@ -13,6 +13,7 @@ self.connection = connection self.using = using self.quote_cache = {} + self.iter_cursor_map = {} def pre_sql_setup(self): """ @@ -677,37 +676,52 @@ resolve_columns = hasattr(self, 'resolve_columns') fields = None has_aggregate_select = bool(self.query.aggregate_select) - for rows in self.execute_sql(MULTI): - for row in rows: - if resolve_columns: - if fields is None: - # We only set this up here because - # related_select_fields isn't populated until - # execute_sql() has been called. - if self.query.select_fields: - fields = self.query.select_fields + self.query.related_select_fields - else: - fields = self.query.model._meta.fields - # If the field was deferred, exclude it from being passed - # into `resolve_columns` because it wasn't selected. - only_load = self.deferred_to_columns() - if only_load: - db_table = self.query.model._meta.db_table - fields = [f for f in fields if db_table in only_load and - f.column in only_load[db_table]] - row = self.resolve_columns(row, fields) + cursor_iter = self.execute_sql(MULTI) + try: + for rows in cursor_iter: + for row in rows: + if resolve_columns: + if fields is None: + # We only set this up here because + # related_select_fields isn't populated until + # execute_sql() has been called. + if self.query.select_fields: + fields = self.query.select_fields + self.query.related_select_fields + else: + fields = self.query.model._meta.fields + # If the field was deferred, exclude it from being passed + # into `resolve_columns` because it wasn't selected. + only_load = self.deferred_to_columns() + if only_load: + db_table = self.query.model._meta.db_table + fields = [f for f in fields if db_table in only_load and + f.column in only_load[db_table]] + row = self.resolve_columns(row, fields) + + if has_aggregate_select: + aggregate_start = len(self.query.extra_select.keys()) + len(self.query.select) + aggregate_end = aggregate_start + len(self.query.aggregate_select) + row = tuple(row[:aggregate_start]) + tuple([ + self.query.resolve_aggregate(value, aggregate, self.connection) + for (alias, aggregate), value + in zip(self.query.aggregate_select.items(), row[aggregate_start:aggregate_end]) + ]) + tuple(row[aggregate_end:]) + + yield row + finally: + # iteration over cursor is completed. + # it's now save to close the cursor and remove it from the lookup dict + if cursor_iter in self.iter_cursor_map: + self.close_cursor_by_iter(cursor_iter) + + def close_cursor_by_iter(self, iter_to_close): + """ + Added method to close iterator from a subclass + """ + if iter_to_close in self.iter_cursor_map: + self.iter_cursor_map[iter_to_close].close() + del self.iter_cursor_map[iter_to_close] - if has_aggregate_select: - aggregate_start = len(self.query.extra_select.keys()) + len(self.query.select) - aggregate_end = aggregate_start + len(self.query.aggregate_select) - row = tuple(row[:aggregate_start]) + tuple([ - self.query.resolve_aggregate(value, aggregate, self.connection) - for (alias, aggregate), value - in zip(self.query.aggregate_select.items(), row[aggregate_start:aggregate_end]) - ]) + tuple(row[aggregate_end:]) - - yield row - def execute_sql(self, result_type=MULTI): """ Run the query against the database and returns the result(s). The @@ -734,12 +748,17 @@ cursor = self.connection.cursor() cursor.execute(sql, params) + # close cursors for single results. if not result_type: return cursor if result_type == SINGLE: if self.query.ordering_aliases: - return cursor.fetchone()[:-len(self.query.ordering_aliases)] - return cursor.fetchone() + temp_result = cursor.fetchone()[:-len(self.query.ordering_aliases)] + cursor.close() + return temp_result + temp_result = cursor.fetchone() + cursor.close() + return temp_result # The MULTI case. if self.query.ordering_aliases: @@ -748,14 +767,18 @@ else: result = iter((lambda: cursor.fetchmany(GET_ITERATOR_CHUNK_SIZE)), self.connection.features.empty_fetchmany_value) + # store iterator only if we use chunks + if self.connection.features.can_use_chunked_reads: + self.iter_cursor_map[result] = cursor if not self.connection.features.can_use_chunked_reads: # If we are using non-chunked reads, we return the same data # structure as normally, but ensure it is all read into memory # before going any further. - return list(result) + temp_ressult = list(result) + cursor.close() + return temp_result return result - class SQLInsertCompiler(SQLCompiler): def placeholder(self, field, val): if field is None: @@ -790,11 +813,17 @@ self.return_id = return_id cursor = super(SQLInsertCompiler, self).execute_sql(None) if not (return_id and cursor): + if cursor: + cursor.close() return if self.connection.features.can_return_id_from_insert: - return self.connection.ops.fetch_returned_insert_id(cursor) - return self.connection.ops.last_insert_id(cursor, + lastid = self.connection.ops.fetch_returned_insert_id(cursor) + cursor.close() + return lastid + result = self.connection.ops.last_insert_id(cursor, self.query.model._meta.db_table, self.query.model._meta.pk.column) + cursor.close() + return result class SQLDeleteCompiler(SQLCompiler): @@ -869,12 +898,19 @@ cursor = super(SQLUpdateCompiler, self).execute_sql(result_type) rows = cursor and cursor.rowcount or 0 is_empty = cursor is None + # explicit close, a Del prob won't trigger immediate GC in Jython + cursor.close() del cursor for query in self.query.get_related_updates(): - aux_rows = query.get_compiler(self.using).execute_sql(result_type) + qcompiler = query.get_compiler(self.using) + aux_rows = qcompiler.execute_sql(result_type) if is_empty: rows = aux_rows is_empty = False + if (result_type is None and type(qcompiler).__name__ not in ['SQLInsertCompiler','SQLUpdateCompiler']): + # Only insert and update compilers are cursor close-safe. + # Other compilers using SQLCompiler.execute_sql(None) will return open cursor. + aux_rows.close() return rows def pre_sql_setup(self): @@ -953,16 +989,20 @@ needs_string_cast = self.connection.features.needs_datetime_string_cast offset = len(self.query.extra_select) - for rows in self.execute_sql(MULTI): - for row in rows: - date = row[offset] - if resolve_columns: - date = self.resolve_columns(row, fields)[offset] - elif needs_string_cast: - date = typecast_timestamp(str(date)) - yield date + date_iter = self.execute_sql(MULTI) + try: + for rows in date_iter: + for row in rows: + date = row[offset] + if resolve_columns: + date = self.resolve_columns(row, fields)[offset] + elif needs_string_cast: + date = typecast_timestamp(str(date)) + yield date + finally: + # request parent to clean up cursor associated with date_iter + self.close_cursor_by_iter(date_iter) - def empty_iter(): """ Returns an iterator containing no results. Index: django/db/models/sql/subqueries.py =================================================================== --- django/db/models/sql/subqueries.py (revision 15147) +++ django/db/models/sql/subqueries.py (working copy) @@ -26,7 +26,9 @@ def do_query(self, table, where, using): self.tables = [table] self.where = where - self.get_compiler(using).execute_sql(None) + cursor = self.get_compiler(using).execute_sql(None) + #parent returns cursor for type None, close it + cursor.close() def delete_batch(self, pk_list, using, field=None): """ @@ -78,7 +80,8 @@ self.where.add((Constraint(None, pk_field.column, pk_field), 'in', pk_list[offset : offset + GET_ITERATOR_CHUNK_SIZE]), AND) - self.get_compiler(using).execute_sql(None) + cursor = self.get_compiler(using).execute_sql(None) + cursor.close() def add_update_values(self, values): """