Ticket #14427: bisect_and_pair.diff

File bisect_and_pair.diff, 7.9 KB (added by russellm, 5 years ago)

Patch implementing bisection and pairing for runtests.py

  • tests/runtests.py

    diff -r a8592432840e tests/runtests.py
    a b  
    11#!/usr/bin/env python
    22
    3 import os, sys, traceback
     3import os, subprocess, sys, traceback
    44import unittest
    55
    66import django.contrib as contrib
     
    8585        self.assert_(not unexpected, "Unexpected Errors: " + '\n'.join(unexpected))
    8686        self.assert_(not missing, "Missing Errors: " + '\n'.join(missing))
    8787
    88 def django_tests(verbosity, interactive, failfast, test_labels):
     88def setup(verbosity, test_labels):
    8989    from django.conf import settings
    90 
    91     old_installed_apps = settings.INSTALLED_APPS
    92     old_root_urlconf = getattr(settings, "ROOT_URLCONF", "")
    93     old_template_dirs = settings.TEMPLATE_DIRS
    94     old_use_i18n = settings.USE_I18N
    95     old_login_url = settings.LOGIN_URL
    96     old_language_code = settings.LANGUAGE_CODE
    97     old_middleware_classes = settings.MIDDLEWARE_CLASSES
     90    state = {
     91        'INSTALLED_APPS': settings.INSTALLED_APPS,
     92        'ROOT_URLCONF': getattr(settings, "ROOT_URLCONF", ""),
     93        'TEMPLATE_DIRS': settings.TEMPLATE_DIRS,
     94        'USE_I18N': settings.USE_I18N,
     95        'LOGIN_URL': settings.LOGIN_URL,
     96        'LANGUAGE_CODE': settings.LANGUAGE_CODE,
     97        'MIDDLEWARE_CLASSES': settings.MIDDLEWARE_CLASSES,
     98    }
    9899
    99100    # Redirect some settings for the duration of these tests.
    100101    settings.INSTALLED_APPS = ALWAYS_INSTALLED_APPS
     
    136137                if model_label not in settings.INSTALLED_APPS:
    137138                    settings.INSTALLED_APPS.append(model_label)
    138139
     140    return state
     141
     142def teardown(state):
     143    from django.conf import settings
     144    # Restore the old settings.
     145    for key, value in state.items():
     146        setattr(settings, key, value)
     147
     148def django_tests(verbosity, interactive, failfast, test_labels):
     149    from django.conf import settings
     150    state = setup(verbosity, test_labels)
     151
    139152    # Add tests for invalid models.
    140153    extra_tests = []
    141154    for model_dir, model_name in get_invalid_models():
     
    169182        test_runner = TestRunner(verbosity=verbosity, interactive=interactive, failfast=failfast)
    170183        failures = test_runner.run_tests(test_labels, extra_tests=extra_tests)
    171184
    172     if failures:
    173         sys.exit(bool(failures))
     185    teardown(state)
     186    return failures
    174187
    175     # Restore the old settings.
    176     settings.INSTALLED_APPS = old_installed_apps
    177     settings.ROOT_URLCONF = old_root_urlconf
    178     settings.TEMPLATE_DIRS = old_template_dirs
    179     settings.USE_I18N = old_use_i18n
    180     settings.LANGUAGE_CODE = old_language_code
    181     settings.LOGIN_URL = old_login_url
    182     settings.MIDDLEWARE_CLASSES = old_middleware_classes
     188
     189def bisect_tests(bisection_label, options, test_labels):
     190    state = setup(int(options.verbosity), test_labels)
     191
     192    if not test_labels:
     193        # Get the full list of test labels to use for bisection
     194        from django.db.models.loading import get_apps
     195        test_labels = [app.__name__.split('.')[-2] for app in get_apps()]
     196
     197    print '***** Bisecting test suite:',' '.join(test_labels)
     198
     199    # Make sure the bisection point isn't in the test list
     200    # Also remove tests that need to be run in specific combinations
     201    for label in [bisection_label, 'model_inheritance_same_model_name']:
     202        try:
     203            test_labels.remove(label)
     204        except ValueError:
     205            pass
     206
     207    subprocess_args = ['python','runtests.py', '--settings=%s' % options.settings]
     208    if options.failfast:
     209        subprocess_args.append('--failfast')
     210    if options.verbosity:
     211        subprocess_args.append('--verbosity=%s' % options.verbosity)
     212    if not options.interactive:
     213        subprocess_args.append('--noinput')
     214
     215    iteration = 1
     216    while len(test_labels) > 1:
     217        midpoint = len(test_labels)/2
     218        test_labels_a = test_labels[:midpoint] + [bisection_label]
     219        test_labels_b = test_labels[midpoint:] + [bisection_label]
     220        print '***** Pass %da: Running the first half of the test suite' % iteration
     221        print '***** Test labels:',' '.join(test_labels_a)
     222        failures_a = subprocess.call(subprocess_args + test_labels_a)
     223
     224        print '***** Pass %db: Running the second half of the test suite' % iteration
     225        print '***** Test labels:',' '.join(test_labels_b)
     226        print
     227        failures_b = subprocess.call(subprocess_args + test_labels_b)
     228
     229        if failures_a and not failures_b:
     230            print "***** Problem found in first half. Bisecting again..."
     231            iteration = iteration + 1
     232            test_labels = test_labels_a[:-1]
     233        elif failures_b and not failures_a:
     234            print "***** Problem found in second half. Bisecting again..."
     235            iteration = iteration + 1
     236            test_labels = test_labels_b[:-1]
     237        elif failures_a and failures_b:
     238            print "***** Multiple sources of failure found"
     239            break
     240        else:
     241            print "***** No source of failure found... try pair execution (--pair)"
     242            break
     243
     244    if len(test_labels) == 1:
     245        print "***** Source of error:",test_labels[0]
     246    teardown(state)
     247
     248def paired_tests(paired_test, options, test_labels):
     249    state = setup(int(options.verbosity), test_labels)
     250
     251    if not test_labels:
     252        print ""
     253        # Get the full list of test labels to use for bisection
     254        from django.db.models.loading import get_apps
     255        test_labels = [app.__name__.split('.')[-2] for app in get_apps()]
     256
     257    print '***** Trying paired execution'
     258
     259    # Make sure the bisection point isn't in the test list
     260    # Also remove tests that need to be run in specific combinations
     261    for label in [paired_test, 'model_inheritance_same_model_name']:
     262        try:
     263            test_labels.remove(label)
     264        except ValueError:
     265            pass
     266
     267    subprocess_args = ['python','runtests.py', '--settings=%s' % options.settings]
     268    if options.failfast:
     269        subprocess_args.append('--failfast')
     270    if options.verbosity:
     271        subprocess_args.append('--verbosity=%s' % options.verbosity)
     272    if not options.interactive:
     273        subprocess_args.append('--noinput')
     274
     275    for i, label in enumerate(test_labels):
     276        print '***** %d of %d: Check test pairing with %s' % (i+1, len(test_labels), label)
     277        failures = subprocess.call(subprocess_args + [label, paired_test])
     278        if failures:
     279            print '***** Found problem pair with',label
     280            return
     281    teardown(state)
    183282
    184283if __name__ == "__main__":
    185284    from optparse import OptionParser
     
    194293        help='Tells Django to stop running the test suite after first failed test.')
    195294    parser.add_option('--settings',
    196295        help='Python path to settings module, e.g. "myproject.settings". If this isn\'t provided, the DJANGO_SETTINGS_MODULE environment variable will be used.')
     296    parser.add_option('--bisect', action='store', dest='bisect', default=None,
     297        help="Bisect the test suite to discover a test that causes a test failure when combined with the named test.")
     298    parser.add_option('--pair', action='store', dest='pair', default=None,
     299        help="Run the test suite in pairs with the named test to find problem pairs.")
    197300    options, args = parser.parse_args()
    198301    if options.settings:
    199302        os.environ['DJANGO_SETTINGS_MODULE'] = options.settings
    200303    elif "DJANGO_SETTINGS_MODULE" not in os.environ:
    201304        parser.error("DJANGO_SETTINGS_MODULE is not set in the environment. "
    202305                      "Set it or use --settings.")
    203     django_tests(int(options.verbosity), options.interactive, options.failfast, args)
     306
     307    if options.bisect:
     308        bisect_tests(options.bisect, options, args)
     309    elif options.pair:
     310        paired_tests(options.pair, options, args)
     311    else:
     312        failures = django_tests(int(options.verbosity), options.interactive, options.failfast, args)
     313        if failures:
     314            sys.exit(bool(failures))
Back to Top