Postgres db backend does not restart connection after disconnect

[lijok]

After a db connection is closed, postgres db backend does not attempt to restart it, causing all further queries to fail.

#
# queries running normally
#

2022-01-05 10:38:57.993171 [info     ] login request using username/password level_number=20 username=admin@axomic.com
2022-01-05 10:38:58.070265 [debug    ] issuing jwt                    claims={'user_uid': 'a12e68b8-2d17-47de-a012-b7d40a979249', 'created': '2022-01-05T10:02:12.654683+00:00', 'email': 'admin@axomic.com', 'last_logged_in': '2022-01-05T10:38:58.063656+00:00', 'updated': '2022-01-05T10:38:58.063677+00:00', 'info': {'admin': True}, 'iat': 1641379138, 'exp': 1641465538, 'iss': 'localhost', 'admin': True} level_number=10
2022-01-05 10:38:58.096159 [info     ] user logged in                 level_number=20 user_email=admin@axomic.com user_uid=a12e68b8-2d17-47de-a012-b7d40a979249

#
# db connection is killed
#

2022-01-05 10:51:53.020392 [info     ] login request using username/password level_number=20 username=admin@axomic.com
Unexpected [OperationalError] raised by servicer method [/axo419.v1.AXO419Service/Login]
Traceback (most recent call last):
  File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/backends/utils.py", line 85, in _execute
    return self.cursor.execute(sql, params)
psycopg2.OperationalError: terminating connection due to administrator command
server closed the connection unexpectedly
	This probably means the server terminated abnormally
	before or while processing the request.


The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi", line 680, in grpc._cython.cygrpc._handle_exceptions
  File "src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi", line 794, in _handle_rpc
  File "src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi", line 545, in _handle_unary_unary_rpc
  File "src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi", line 402, in _finish_handler_with_unary_response
  File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/axomic_python_sdk/grpc/_aio_jwt_authentication_interceptor.py", line 130, in new_request_handler
    return await request_handler(request_or_iterator, new_context)
  File "/Users/user/repos/AXO419/axo419/api/axo419_v1_service_servicer.py", line 31, in Login
    token, expires, user = await core.login(
  File "/Users/user/repos/AXO419/axo419/core/auth.py", line 22, in login
    usr = await user.get_user(email=username)
  File "/Users/user/repos/AXO419/axo419/core/user.py", line 75, in get_user
    users = await search_users(email=email, first=1)
  File "/Users/user/repos/AXO419/axo419/core/user.py", line 53, in search_users
    page = await db.search_users(
  File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/asgiref/sync.py", line 444, in __call__
    ret = await asyncio.wait_for(future, timeout=None)
  File "/usr/local/Cellar/python@3.10/3.10.0_2/Frameworks/Python.framework/Versions/3.10/lib/python3.10/asyncio/tasks.py", line 408, in wait_for
    return await fut
  File "/usr/local/Cellar/python@3.10/3.10.0_2/Frameworks/Python.framework/Versions/3.10/lib/python3.10/concurrent/futures/thread.py", line 52, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/asgiref/sync.py", line 486, in thread_handler
    return func(*args, **kwargs)
  File "/Users/user/repos/AXO419/axo419/db/user.py", line 54, in search_users
    page = axomic_python_sdk.pagination.paginate(
  File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/axomic_python_sdk/pagination/_paginate.py", line 49, in paginate
    page = pagination._limit_pagination.first_pagination(
  File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/axomic_python_sdk/pagination/_limit_pagination.py", line 12, in first_pagination
    total_count = queryset.count()
  File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/models/query.py", line 416, in count
    return self.query.get_count(using=self.db)
  File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/models/sql/query.py", line 516, in get_count
    number = obj.get_aggregation(using, ['__count'])['__count']
  File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/models/sql/query.py", line 501, in get_aggregation
    result = compiler.execute_sql(SINGLE)
  File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/models/sql/compiler.py", line 1202, in execute_sql
    cursor.execute(sql, params)
  File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/backends/utils.py", line 99, in execute
    return super().execute(sql, params)
  File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/backends/utils.py", line 67, in execute
    return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
  File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/backends/utils.py", line 76, in _execute_with_wrappers
    return executor(sql, params, many, context)
  File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/backends/utils.py", line 80, in _execute
    with self.db.wrap_database_errors:
  File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/utils.py", line 90, in __exit__
    raise dj_exc_value.with_traceback(traceback) from exc_value
  File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/backends/utils.py", line 85, in _execute
    return self.cursor.execute(sql, params)
django.db.utils.OperationalError: terminating connection due to administrator command
server closed the connection unexpectedly
	This probably means the server terminated abnormally
	before or while processing the request.

2022-01-05 10:58:35.678493 [info     ] login request using username/password level_number=20 username=admin@axomic.com
Unexpected [InterfaceError] raised by servicer method [/axo419.v1.AXO419Service/Login]
Traceback (most recent call last):
  File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/backends/base/base.py", line 248, in _cursor
    return self._prepare_cursor(self.create_cursor(name))
  File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/utils/asyncio.py", line 25, in inner
    return func(*args, **kwargs)
  File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/backends/postgresql/base.py", line 248, in create_cursor
    cursor = self.connection.cursor()
psycopg2.InterfaceError: connection already closed

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi", line 680, in grpc._cython.cygrpc._handle_exceptions
  File "src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi", line 794, in _handle_rpc
  File "src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi", line 545, in _handle_unary_unary_rpc
  File "src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi", line 402, in _finish_handler_with_unary_response
  File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/axomic_python_sdk/grpc/_aio_jwt_authentication_interceptor.py", line 130, in new_request_handler
    return await request_handler(request_or_iterator, new_context)
  File "/Users/user/repos/AXO419/axo419/api/axo419_v1_service_servicer.py", line 31, in Login
    token, expires, user = await core.login(
  File "/Users/user/repos/AXO419/axo419/core/auth.py", line 22, in login
    usr = await user.get_user(email=username)
  File "/Users/user/repos/AXO419/axo419/core/user.py", line 75, in get_user
    users = await search_users(email=email, first=1)
  File "/Users/user/repos/AXO419/axo419/core/user.py", line 53, in search_users
    page = await db.search_users(
  File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/asgiref/sync.py", line 444, in __call__
    ret = await asyncio.wait_for(future, timeout=None)
  File "/usr/local/Cellar/python@3.10/3.10.0_2/Frameworks/Python.framework/Versions/3.10/lib/python3.10/asyncio/tasks.py", line 408, in wait_for
    return await fut
  File "/usr/local/Cellar/python@3.10/3.10.0_2/Frameworks/Python.framework/Versions/3.10/lib/python3.10/concurrent/futures/thread.py", line 52, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/asgiref/sync.py", line 486, in thread_handler
    return func(*args, **kwargs)
  File "/Users/user/repos/AXO419/axo419/db/user.py", line 54, in search_users
    page = axomic_python_sdk.pagination.paginate(
  File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/axomic_python_sdk/pagination/_paginate.py", line 49, in paginate
    page = pagination._limit_pagination.first_pagination(
  File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/axomic_python_sdk/pagination/_limit_pagination.py", line 12, in first_pagination
    total_count = queryset.count()
  File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/models/query.py", line 416, in count
    return self.query.get_count(using=self.db)
  File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/models/sql/query.py", line 516, in get_count
    number = obj.get_aggregation(using, ['__count'])['__count']
  File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/models/sql/query.py", line 501, in get_aggregation
    result = compiler.execute_sql(SINGLE)
  File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/models/sql/compiler.py", line 1200, in execute_sql
    cursor = self.connection.cursor()
  File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/utils/asyncio.py", line 25, in inner
    return func(*args, **kwargs)
  File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/backends/base/base.py", line 272, in cursor
    return self._cursor()
  File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/backends/base/base.py", line 247, in _cursor
    with self.wrap_database_errors:
  File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/utils.py", line 90, in __exit__
    raise dj_exc_value.with_traceback(traceback) from exc_value
  File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/backends/base/base.py", line 248, in _cursor
    return self._prepare_cursor(self.create_cursor(name))
  File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/utils/asyncio.py", line 25, in inner
    return func(*args, **kwargs)
  File "/Users/user/repos/AXO419/.venv/lib/python3.10/site-packages/django/db/backends/postgresql/base.py", line 248, in create_cursor
    cursor = self.connection.cursor()
django.db.utils.InterfaceError: connection already closed

Note that we're using django's ORM standalone

[Mariusz Felisiak]

Duplicate of #24810 and #30398.