Ticket #20099: 0001-Implement-BrokenLinkEmailsMiddleware.is_request_we_s.2.patch

django/middleware/common.py

@@ -137 +137 @@
         """
         Send broken link emails for relevant 404 NOT FOUND responses.
         """
-        if response.status_code == 404 and not settings.DEBUG:
-            domain = request.get_host()
-            path = request.get_full_path()
-            referer = request.META.get('HTTP_REFERER', '')
-            is_internal = self.is_internal_request(domain, referer)
-            is_not_search_engine = '?' not in referer
-            is_ignorable = self.is_ignorable_404(path)
-            if referer and (is_internal or is_not_search_engine) and not is_ignorable:
-                ua = request.META.get('HTTP_USER_AGENT', '<none>')
-                ip = request.META.get('REMOTE_ADDR', '<none>')
-                mail_managers(
-                    "Broken %slink on %s" % (('INTERNAL ' if is_internal else ''), domain),
-                    "Referrer: %s\nRequested URL: %s\nUser agent: %s\nIP address: %s\n" % (referer, path, ua, ip),
-                    fail_silently=True)
+        
+        if response.status_code == 404 and not settings.DEBUG and \
+                                 self.is_request_we_should_notify_for(request):
+            is_internal = self.is_internal_request(request)
+            mail_managers(
+                "Broken %slink on %s" % (
+                    ('INTERNAL ' if is_internal else ''),
+                    request.get_host()
+                ),
+                "Referrer: %s\nRequested URL: %s\nUser agent: %s\n"
+                "IP address: %s\n" % (
+                    request.META.get('HTTP_REFERER', ''),
+                    request.get_full_path(),
+                    request.META.get('HTTP_USER_AGENT', ''),
+                    request.META.get('REMOTE_ADDR', '')
+                ),
+                fail_silently=True)
         return response
 
-    def is_internal_request(self, domain, referer):
+    def is_internal_request(self, request):
         """
         Returns True if the referring URL is the same domain as the current request.
         """
         # Different subdomains are treated as different domains.
+        referer = request.META.get('HTTP_REFERER', '')
+        domain = request.get_host()
         return re.match("^https?://%s/" % re.escape(domain), referer)
 
-    def is_ignorable_404(self, uri):
+    def is_request_we_should_notify_for(self, request):
         """
-        Returns True if a 404 at the given URL *shouldn't* notify the site managers.
+        Should we notify that `request` resulted in a 404 error?
+        
+        This depends on the URL, the referer, and may be subclassed to check
+        other things.
         """
-        return any(pattern.search(uri) for pattern in settings.IGNORABLE_404_URLS)
+        referer = request.META.get('HTTP_REFERER', '')
+        
+        if not referer:
+            return False
+        
+        is_internal = self.is_internal_request(request)
+        is_search_engine = '?' in referer
+        
+        if not is_internal and is_search_engine:
+            return False
+        
+        return not any(pattern.search(request.get_full_path()) for pattern in
+                       settings.IGNORABLE_404_URLS)

docs/howto/error-reporting.txt

@@ -98 +98 @@
 (Note that these are regular expressions, so we put a backslash in front of
 periods to escape them.)
 
+If you'd like to customize the behavior of
+:class:`django.middleware.common.BrokenLinkEmailsMiddleware` further (for
+example to ignore requests coming from web crawlers), you should subclass it
+and override its methods.
+
 .. seealso::
 
    404 errors are logged using the logging framework. By default, these log

tests/middleware/tests.py

@@ -320 +320 @@
         BrokenLinkEmailsMiddleware().process_response(self.req, self.resp)
         self.assertEqual(len(mail.outbox), 0)
 
+    def test_custom_request_checker(self):
+        class SubclassedMiddleware(BrokenLinkEmailsMiddleware):
+            ignored_user_agent_patterns = (re.compile(r'Spider.*'),
+                                           re.compile(r'Robot.*'))
+            def is_request_we_should_notify_for(self, request):
+                '''Check user-agent in addition to normal checks.'''
+                assert isinstance(request, HttpRequest)
+                if not super(SubclassedMiddleware, self). \
+                                      is_request_we_should_notify_for(request):
+                    return False
+                user_agent = request.META['HTTP_USER_AGENT']
+                return not any(pattern.search(user_agent) for pattern in
+                               self.ignored_user_agent_patterns)
 
+        self.req.META['HTTP_REFERER'] = '/another/url/'
+        self.req.META['HTTP_USER_AGENT'] = 'Spider machine 3.4'
+        SubclassedMiddleware().process_response(self.req, self.resp)
+        self.assertEqual(len(mail.outbox), 0)
+        self.req.META['HTTP_USER_AGENT'] = 'My user agent'
+        SubclassedMiddleware().process_response(self.req, self.resp)
+        self.assertEqual(len(mail.outbox), 1)
+    
 class ConditionalGetMiddlewareTest(TestCase):
     urls = 'middleware.cond_get_urls'
     def setUp(self):