Code

Ticket #14427: bisect_and_pair.diff

File bisect_and_pair.diff, 7.9 KB (added by russellm, 4 years ago)

Patch implementing bisection and pairing for runtests.py

Line 
1diff -r a8592432840e tests/runtests.py
2--- a/tests/runtests.py Sat Oct 09 08:12:50 2010 +0000
3+++ b/tests/runtests.py Sat Oct 09 22:23:01 2010 +0800
4@@ -1,6 +1,6 @@
5 #!/usr/bin/env python
6 
7-import os, sys, traceback
8+import os, subprocess, sys, traceback
9 import unittest
10 
11 import django.contrib as contrib
12@@ -85,16 +85,17 @@
13         self.assert_(not unexpected, "Unexpected Errors: " + '\n'.join(unexpected))
14         self.assert_(not missing, "Missing Errors: " + '\n'.join(missing))
15 
16-def django_tests(verbosity, interactive, failfast, test_labels):
17+def setup(verbosity, test_labels):
18     from django.conf import settings
19-
20-    old_installed_apps = settings.INSTALLED_APPS
21-    old_root_urlconf = getattr(settings, "ROOT_URLCONF", "")
22-    old_template_dirs = settings.TEMPLATE_DIRS
23-    old_use_i18n = settings.USE_I18N
24-    old_login_url = settings.LOGIN_URL
25-    old_language_code = settings.LANGUAGE_CODE
26-    old_middleware_classes = settings.MIDDLEWARE_CLASSES
27+    state = {
28+        'INSTALLED_APPS': settings.INSTALLED_APPS,
29+        'ROOT_URLCONF': getattr(settings, "ROOT_URLCONF", ""),
30+        'TEMPLATE_DIRS': settings.TEMPLATE_DIRS,
31+        'USE_I18N': settings.USE_I18N,
32+        'LOGIN_URL': settings.LOGIN_URL,
33+        'LANGUAGE_CODE': settings.LANGUAGE_CODE,
34+        'MIDDLEWARE_CLASSES': settings.MIDDLEWARE_CLASSES,
35+    }
36 
37     # Redirect some settings for the duration of these tests.
38     settings.INSTALLED_APPS = ALWAYS_INSTALLED_APPS
39@@ -136,6 +137,18 @@
40                 if model_label not in settings.INSTALLED_APPS:
41                     settings.INSTALLED_APPS.append(model_label)
42 
43+    return state
44+
45+def teardown(state):
46+    from django.conf import settings
47+    # Restore the old settings.
48+    for key, value in state.items():
49+        setattr(settings, key, value)
50+
51+def django_tests(verbosity, interactive, failfast, test_labels):
52+    from django.conf import settings
53+    state = setup(verbosity, test_labels)
54+
55     # Add tests for invalid models.
56     extra_tests = []
57     for model_dir, model_name in get_invalid_models():
58@@ -169,17 +182,103 @@
59         test_runner = TestRunner(verbosity=verbosity, interactive=interactive, failfast=failfast)
60         failures = test_runner.run_tests(test_labels, extra_tests=extra_tests)
61 
62-    if failures:
63-        sys.exit(bool(failures))
64+    teardown(state)
65+    return failures
66 
67-    # Restore the old settings.
68-    settings.INSTALLED_APPS = old_installed_apps
69-    settings.ROOT_URLCONF = old_root_urlconf
70-    settings.TEMPLATE_DIRS = old_template_dirs
71-    settings.USE_I18N = old_use_i18n
72-    settings.LANGUAGE_CODE = old_language_code
73-    settings.LOGIN_URL = old_login_url
74-    settings.MIDDLEWARE_CLASSES = old_middleware_classes
75+
76+def bisect_tests(bisection_label, options, test_labels):
77+    state = setup(int(options.verbosity), test_labels)
78+
79+    if not test_labels:
80+        # Get the full list of test labels to use for bisection
81+        from django.db.models.loading import get_apps
82+        test_labels = [app.__name__.split('.')[-2] for app in get_apps()]
83+
84+    print '***** Bisecting test suite:',' '.join(test_labels)
85+
86+    # Make sure the bisection point isn't in the test list
87+    # Also remove tests that need to be run in specific combinations
88+    for label in [bisection_label, 'model_inheritance_same_model_name']:
89+        try:
90+            test_labels.remove(label)
91+        except ValueError:
92+            pass
93+
94+    subprocess_args = ['python','runtests.py', '--settings=%s' % options.settings]
95+    if options.failfast:
96+        subprocess_args.append('--failfast')
97+    if options.verbosity:
98+        subprocess_args.append('--verbosity=%s' % options.verbosity)
99+    if not options.interactive:
100+        subprocess_args.append('--noinput')
101+
102+    iteration = 1
103+    while len(test_labels) > 1:
104+        midpoint = len(test_labels)/2
105+        test_labels_a = test_labels[:midpoint] + [bisection_label]
106+        test_labels_b = test_labels[midpoint:] + [bisection_label]
107+        print '***** Pass %da: Running the first half of the test suite' % iteration
108+        print '***** Test labels:',' '.join(test_labels_a)
109+        failures_a = subprocess.call(subprocess_args + test_labels_a)
110+
111+        print '***** Pass %db: Running the second half of the test suite' % iteration
112+        print '***** Test labels:',' '.join(test_labels_b)
113+        print
114+        failures_b = subprocess.call(subprocess_args + test_labels_b)
115+
116+        if failures_a and not failures_b:
117+            print "***** Problem found in first half. Bisecting again..."
118+            iteration = iteration + 1
119+            test_labels = test_labels_a[:-1]
120+        elif failures_b and not failures_a:
121+            print "***** Problem found in second half. Bisecting again..."
122+            iteration = iteration + 1
123+            test_labels = test_labels_b[:-1]
124+        elif failures_a and failures_b:
125+            print "***** Multiple sources of failure found"
126+            break
127+        else:
128+            print "***** No source of failure found... try pair execution (--pair)"
129+            break
130+
131+    if len(test_labels) == 1:
132+        print "***** Source of error:",test_labels[0]
133+    teardown(state)
134+
135+def paired_tests(paired_test, options, test_labels):
136+    state = setup(int(options.verbosity), test_labels)
137+
138+    if not test_labels:
139+        print ""
140+        # Get the full list of test labels to use for bisection
141+        from django.db.models.loading import get_apps
142+        test_labels = [app.__name__.split('.')[-2] for app in get_apps()]
143+
144+    print '***** Trying paired execution'
145+
146+    # Make sure the bisection point isn't in the test list
147+    # Also remove tests that need to be run in specific combinations
148+    for label in [paired_test, 'model_inheritance_same_model_name']:
149+        try:
150+            test_labels.remove(label)
151+        except ValueError:
152+            pass
153+
154+    subprocess_args = ['python','runtests.py', '--settings=%s' % options.settings]
155+    if options.failfast:
156+        subprocess_args.append('--failfast')
157+    if options.verbosity:
158+        subprocess_args.append('--verbosity=%s' % options.verbosity)
159+    if not options.interactive:
160+        subprocess_args.append('--noinput')
161+
162+    for i, label in enumerate(test_labels):
163+        print '***** %d of %d: Check test pairing with %s' % (i+1, len(test_labels), label)
164+        failures = subprocess.call(subprocess_args + [label, paired_test])
165+        if failures:
166+            print '***** Found problem pair with',label
167+            return
168+    teardown(state)
169 
170 if __name__ == "__main__":
171     from optparse import OptionParser
172@@ -194,10 +293,22 @@
173         help='Tells Django to stop running the test suite after first failed test.')
174     parser.add_option('--settings',
175         help='Python path to settings module, e.g. "myproject.settings". If this isn\'t provided, the DJANGO_SETTINGS_MODULE environment variable will be used.')
176+    parser.add_option('--bisect', action='store', dest='bisect', default=None,
177+        help="Bisect the test suite to discover a test that causes a test failure when combined with the named test.")
178+    parser.add_option('--pair', action='store', dest='pair', default=None,
179+        help="Run the test suite in pairs with the named test to find problem pairs.")
180     options, args = parser.parse_args()
181     if options.settings:
182         os.environ['DJANGO_SETTINGS_MODULE'] = options.settings
183     elif "DJANGO_SETTINGS_MODULE" not in os.environ:
184         parser.error("DJANGO_SETTINGS_MODULE is not set in the environment. "
185                       "Set it or use --settings.")
186-    django_tests(int(options.verbosity), options.interactive, options.failfast, args)
187+
188+    if options.bisect:
189+        bisect_tests(options.bisect, options, args)
190+    elif options.pair:
191+        paired_tests(options.pair, options, args)
192+    else:
193+        failures = django_tests(int(options.verbosity), options.interactive, options.failfast, args)
194+        if failures:
195+            sys.exit(bool(failures))