diff --git a/django/contrib/sessions/middleware.py b/django/contrib/sessions/middleware.py
index 6795354..f028b12 100644
--- a/django/contrib/sessions/middleware.py
+++ b/django/contrib/sessions/middleware.py
@@ -21,11 +21,16 @@ class SessionMiddleware(MiddlewareMixin):
 
     def process_response(self, request, response):
         """
-        If request.session was modified, or if the configuration is to save the
-        session every time, save the changes and set a session cookie or delete
-        the session cookie if the session has been emptied.
+        If the request contains the session cookie but the session is empty we
+        should remove the cookie. If the session is modified or set to always
+        save we should prepare the cookie headers, check that we are not 
+        returning a 500 response, save the session, and set the cookie. Last if
+        the session is not empty but we did not need to set the cookie because
+        it was not modified or our settings do not require us to always save we
+        can assume it was previously set and just apply the vary cookie header.
         """
         try:
+            # TODO is this accessed property needed in this middleware?
             accessed = request.session.accessed
             modified = request.session.modified
             empty = request.session.is_empty()
@@ -33,42 +38,47 @@ class SessionMiddleware(MiddlewareMixin):
             pass
         else:
             # First check if we need to delete this cookie.
-            # The session should be deleted only if the session is entirely empty
+            # The session cookie should be deleted only if the session is
+            # entirely empty.
             if settings.SESSION_COOKIE_NAME in request.COOKIES and empty:
                 response.delete_cookie(
                     settings.SESSION_COOKIE_NAME,
                     path=settings.SESSION_COOKIE_PATH,
                     domain=settings.SESSION_COOKIE_DOMAIN,
                 )
-            else:
-                if accessed:
-                    patch_vary_headers(response, ('Cookie',))
-                if (modified or settings.SESSION_SAVE_EVERY_REQUEST) and not empty:
-                    if request.session.get_expire_at_browser_close():
-                        max_age = None
-                        expires = None
-                    else:
-                        max_age = request.session.get_expiry_age()
-                        expires_time = time.time() + max_age
-                        expires = http_date(expires_time)
-                    # Save the session data and refresh the client cookie.
-                    # Skip session save for 500 responses, refs #3881.
-                    if response.status_code != 500:
-                        try:
-                            request.session.save()
-                        except UpdateError:
-                            raise SuspiciousOperation(
-                                "The request's session was deleted before the "
-                                "request completed. The user may have logged "
-                                "out in a concurrent request, for example."
-                            )
-                        response.set_cookie(
-                            settings.SESSION_COOKIE_NAME,
-                            request.session.session_key, max_age=max_age,
-                            expires=expires, domain=settings.SESSION_COOKIE_DOMAIN,
-                            path=settings.SESSION_COOKIE_PATH,
-                            secure=settings.SESSION_COOKIE_SECURE or None,
-                            httponly=settings.SESSION_COOKIE_HTTPONLY or None,
-                            samesite=settings.SESSION_COOKIE_SAMESITE,
+                patch_vary_headers(response, ('Cookie',))
+            elif (
+                (modified or settings.SESSION_SAVE_EVERY_REQUEST) and
+                not empty
+            ):
+                if request.session.get_expire_at_browser_close():
+                    max_age = None
+                    expires = None
+                else:
+                    max_age = request.session.get_expiry_age()
+                    expires_time = time.time() + max_age
+                    expires = http_date(expires_time)
+                # Save the session data and refresh the client cookie.
+                # Skip session save for 500 responses, refs #3881.
+                if response.status_code != 500:
+                    try:
+                        request.session.save()
+                    except UpdateError:
+                        raise SuspiciousOperation(
+                            "The request's session was deleted before the "
+                            "request completed. The user may have logged "
+                            "out in a concurrent request, for example."
                         )
+                    response.set_cookie(
+                        settings.SESSION_COOKIE_NAME,
+                        request.session.session_key, max_age=max_age,
+                        expires=expires, domain=settings.SESSION_COOKIE_DOMAIN,
+                        path=settings.SESSION_COOKIE_PATH,
+                        secure=settings.SESSION_COOKIE_SECURE or None,
+                        httponly=settings.SESSION_COOKIE_HTTPONLY or None,
+                        samesite=settings.SESSION_COOKIE_SAMESITE,
+                    )
+                    patch_vary_headers(response, ('Cookie',))
+            elif not empty:
+                patch_vary_headers(response, ('Cookie',))
         return response
