Opened 3 years ago
Closed 3 years ago
#33412 closed Bug (duplicate)
Postgres db backend does not restart connection after disconnect
Reported by: | lijok | Owned by: | nobody |
---|---|---|---|
Component: | Database layer (models, ORM) | Version: | 4.0 |
Severity: | Normal | Keywords: | postgres db |
Cc: | Triage Stage: | Unreviewed | |
Has patch: | no | Needs documentation: | no |
Needs tests: | no | Patch needs improvement: | no |
Easy pickings: | no | UI/UX: | no |
Description
After a db connection is closed, postgres db backend does not attempt to restart it, causing all further queries to fail.
# # queries running normally # 2022-01-05 10:38:57.993171 [info ] login request using username/password level_number=20 username=admin@axomic.com 2022-01-05 10:38:58.070265 [debug ] issuing jwt claims={'user_uid': 'a12e68b8-2d17-47de-a012-b7d40a979249', 'created': '2022-01-05T10:02:12.654683+00:00', 'email': 'admin@axomic.com', 'last_logged_in': '2022-01-05T10:38:58.063656+00:00', 'updated': '2022-01-05T10:38:58.063677+00:00', 'info': {'admin': True}, 'iat': 1641379138, 'exp': 1641465538, 'iss': 'localhost', 'admin': True} level_number=10 2022-01-05 10:38:58.096159 [info ] user logged in level_number=20 user_email=admin@axomic.com user_uid=a12e68b8-2d17-47de-a012-b7d40a979249 # # db connection is killed # 2022-01-05 10:51:53.020392 [info ] login request using username/password level_number=20 username=admin@axomic.com Unexpected [OperationalError] raised by servicer method [/axo419.v1.AXO419Service/Login] Traceback (most recent call last): File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/backends/utils.py", line 85, in _execute return self.cursor.execute(sql, params) psycopg2.OperationalError: terminating connection due to administrator command server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. The above exception was the direct cause of the following exception: Traceback (most recent call last): File "src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi", line 680, in grpc._cython.cygrpc._handle_exceptions File "src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi", line 794, in _handle_rpc File "src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi", line 545, in _handle_unary_unary_rpc File "src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi", line 402, in _finish_handler_with_unary_response File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/axomic_python_sdk/grpc/_aio_jwt_authentication_interceptor.py", line 130, in new_request_handler return await request_handler(request_or_iterator, new_context) File "/Users/user/repos/AXO419/axo419/api/axo419_v1_service_servicer.py", line 31, in Login token, expires, user = await core.login( File "/Users/user/repos/AXO419/axo419/core/auth.py", line 22, in login usr = await user.get_user(email=username) File "/Users/user/repos/AXO419/axo419/core/user.py", line 75, in get_user users = await search_users(email=email, first=1) File "/Users/user/repos/AXO419/axo419/core/user.py", line 53, in search_users page = await db.search_users( File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/asgiref/sync.py", line 444, in __call__ ret = await asyncio.wait_for(future, timeout=None) File "/usr/local/Cellar/python@3.10/3.10.0_2/Frameworks/Python.framework/Versions/3.10/lib/python3.10/asyncio/tasks.py", line 408, in wait_for return await fut File "/usr/local/Cellar/python@3.10/3.10.0_2/Frameworks/Python.framework/Versions/3.10/lib/python3.10/concurrent/futures/thread.py", line 52, in run result = self.fn(*self.args, **self.kwargs) File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/asgiref/sync.py", line 486, in thread_handler return func(*args, **kwargs) File "/Users/user/repos/AXO419/axo419/db/user.py", line 54, in search_users page = axomic_python_sdk.pagination.paginate( File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/axomic_python_sdk/pagination/_paginate.py", line 49, in paginate page = pagination._limit_pagination.first_pagination( File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/axomic_python_sdk/pagination/_limit_pagination.py", line 12, in first_pagination total_count = queryset.count() File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/models/query.py", line 416, in count return self.query.get_count(using=self.db) File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/models/sql/query.py", line 516, in get_count number = obj.get_aggregation(using, ['__count'])['__count'] File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/models/sql/query.py", line 501, in get_aggregation result = compiler.execute_sql(SINGLE) File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/models/sql/compiler.py", line 1202, in execute_sql cursor.execute(sql, params) File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/backends/utils.py", line 99, in execute return super().execute(sql, params) File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/backends/utils.py", line 67, in execute return self._execute_with_wrappers(sql, params, many=False, executor=self._execute) File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/backends/utils.py", line 76, in _execute_with_wrappers return executor(sql, params, many, context) File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/backends/utils.py", line 80, in _execute with self.db.wrap_database_errors: File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/utils.py", line 90, in __exit__ raise dj_exc_value.with_traceback(traceback) from exc_value File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/backends/utils.py", line 85, in _execute return self.cursor.execute(sql, params) django.db.utils.OperationalError: terminating connection due to administrator command server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2022-01-05 10:58:35.678493 [info ] login request using username/password level_number=20 username=admin@axomic.com Unexpected [InterfaceError] raised by servicer method [/axo419.v1.AXO419Service/Login] Traceback (most recent call last): File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/backends/base/base.py", line 248, in _cursor return self._prepare_cursor(self.create_cursor(name)) File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/utils/asyncio.py", line 25, in inner return func(*args, **kwargs) File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/backends/postgresql/base.py", line 248, in create_cursor cursor = self.connection.cursor() psycopg2.InterfaceError: connection already closed The above exception was the direct cause of the following exception: Traceback (most recent call last): File "src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi", line 680, in grpc._cython.cygrpc._handle_exceptions File "src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi", line 794, in _handle_rpc File "src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi", line 545, in _handle_unary_unary_rpc File "src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi", line 402, in _finish_handler_with_unary_response File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/axomic_python_sdk/grpc/_aio_jwt_authentication_interceptor.py", line 130, in new_request_handler return await request_handler(request_or_iterator, new_context) File "/Users/user/repos/AXO419/axo419/api/axo419_v1_service_servicer.py", line 31, in Login token, expires, user = await core.login( File "/Users/user/repos/AXO419/axo419/core/auth.py", line 22, in login usr = await user.get_user(email=username) File "/Users/user/repos/AXO419/axo419/core/user.py", line 75, in get_user users = await search_users(email=email, first=1) File "/Users/user/repos/AXO419/axo419/core/user.py", line 53, in search_users page = await db.search_users( File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/asgiref/sync.py", line 444, in __call__ ret = await asyncio.wait_for(future, timeout=None) File "/usr/local/Cellar/python@3.10/3.10.0_2/Frameworks/Python.framework/Versions/3.10/lib/python3.10/asyncio/tasks.py", line 408, in wait_for return await fut File "/usr/local/Cellar/python@3.10/3.10.0_2/Frameworks/Python.framework/Versions/3.10/lib/python3.10/concurrent/futures/thread.py", line 52, in run result = self.fn(*self.args, **self.kwargs) File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/asgiref/sync.py", line 486, in thread_handler return func(*args, **kwargs) File "/Users/user/repos/AXO419/axo419/db/user.py", line 54, in search_users page = axomic_python_sdk.pagination.paginate( File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/axomic_python_sdk/pagination/_paginate.py", line 49, in paginate page = pagination._limit_pagination.first_pagination( File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/axomic_python_sdk/pagination/_limit_pagination.py", line 12, in first_pagination total_count = queryset.count() File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/models/query.py", line 416, in count return self.query.get_count(using=self.db) File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/models/sql/query.py", line 516, in get_count number = obj.get_aggregation(using, ['__count'])['__count'] File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/models/sql/query.py", line 501, in get_aggregation result = compiler.execute_sql(SINGLE) File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/models/sql/compiler.py", line 1200, in execute_sql cursor = self.connection.cursor() File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/utils/asyncio.py", line 25, in inner return func(*args, **kwargs) File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/backends/base/base.py", line 272, in cursor return self._cursor() File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/backends/base/base.py", line 247, in _cursor with self.wrap_database_errors: File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/utils.py", line 90, in __exit__ raise dj_exc_value.with_traceback(traceback) from exc_value File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/backends/base/base.py", line 248, in _cursor return self._prepare_cursor(self.create_cursor(name)) File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/utils/asyncio.py", line 25, in inner return func(*args, **kwargs) File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/backends/postgresql/base.py", line 248, in create_cursor cursor = self.connection.cursor() django.db.utils.InterfaceError: connection already closed
Note that we're using django's ORM standalone
Note:
See TracTickets
for help on using tickets.
Duplicate of #24810 and #30398.