755
754
# want to assume that thats *all* that would happen.
756
755
self.overrideAttr(bzrlib.version, '_get_bzr_source_tree', lambda: None)
758
def test_assigned_benchmark_file_stores_date(self):
759
self._patch_get_bzr_source_tree()
761
result = bzrlib.tests.TextTestResult(self._log_file,
766
output_string = output.getvalue()
767
# if you are wondering about the regexp please read the comment in
768
# test_bench_history (bzrlib.tests.test_selftest.TestRunner)
769
# XXX: what comment? -- Andrew Bennetts
770
self.assertContainsRe(output_string, "--date [0-9.]+")
772
def test_benchhistory_records_test_times(self):
773
self._patch_get_bzr_source_tree()
774
result_stream = StringIO()
775
result = bzrlib.tests.TextTestResult(
779
bench_history=result_stream
782
# we want profile a call and check that its test duration is recorded
783
# make a new test instance that when run will generate a benchmark
784
example_test_case = TestTestResult("_time_hello_world_encoding")
785
# execute the test, which should succeed and record times
786
example_test_case.run(result)
787
lines = result_stream.getvalue().splitlines()
788
self.assertEqual(2, len(lines))
789
self.assertContainsRe(lines[1],
790
" *[0-9]+ms bzrlib.tests.test_selftest.TestTestResult"
791
"._time_hello_world_encoding")
793
757
def _time_hello_world_encoding(self):
794
758
"""Profile two sleep calls
835
799
self.assertContainsRe(output,
836
800
r"LSProf output for <type 'unicode'>\(\('world',\), {'errors': 'replace'}\)\n")
802
def test_uses_time_from_testtools(self):
803
"""Test case timings in verbose results should use testtools times"""
805
class TimeAddedVerboseTestResult(tests.VerboseTestResult):
806
def startTest(self, test):
807
self.time(datetime.datetime.utcfromtimestamp(1.145))
808
super(TimeAddedVerboseTestResult, self).startTest(test)
809
def addSuccess(self, test):
810
self.time(datetime.datetime.utcfromtimestamp(51.147))
811
super(TimeAddedVerboseTestResult, self).addSuccess(test)
812
def report_tests_starting(self): pass
814
self.get_passing_test().run(TimeAddedVerboseTestResult(sio, 0, 2))
815
self.assertEndsWith(sio.getvalue(), "OK 50002ms\n")
838
817
def test_known_failure(self):
839
818
"""A KnownFailure being raised should trigger several result actions."""
840
819
class InstrumentedTestResult(tests.ExtendedTestResult):
841
820
def stopTestRun(self): pass
842
def startTests(self): pass
843
def report_test_start(self, test): pass
821
def report_tests_starting(self): pass
844
822
def report_known_failure(self, test, err=None, details=None):
845
823
self._call = test, 'known failure'
846
824
result = InstrumentedTestResult(None, None, None, None)
1200
1188
self.overrideAttr(bzrlib.version, '_get_bzr_source_tree', new_get)
1202
def test_bench_history(self):
1203
# tests that the running the benchmark passes bench_history into
1204
# the test result object. We can tell that happens if
1205
# _get_bzr_source_tree is called.
1206
self._patch_get_bzr_source_tree()
1207
test = TestRunner('dummy_test')
1209
runner = tests.TextTestRunner(stream=self._log_file,
1210
bench_history=output)
1211
result = self.run_test_runner(runner, test)
1212
output_string = output.getvalue()
1213
self.assertContainsRe(output_string, "--date [0-9.]+")
1214
self.assertLength(1, self._get_source_tree_calls)
1190
def test_verbose_test_count(self):
1191
"""A verbose test run reports the right test count at the start"""
1192
suite = TestUtil.TestSuite([
1193
unittest.FunctionTestCase(lambda:None),
1194
unittest.FunctionTestCase(lambda:None)])
1195
self.assertEqual(suite.countTestCases(), 2)
1197
runner = tests.TextTestRunner(stream=stream, verbosity=2)
1198
# Need to use the CountingDecorator as that's what sets num_tests
1199
result = self.run_test_runner(runner, tests.CountingDecorator(suite))
1200
self.assertStartsWith(stream.getvalue(), "running 2 tests")
1216
1202
def test_startTestRun(self):
1217
1203
"""run should call result.startTestRun()"""
1219
class LoggingDecorator(tests.ForwardingResult):
1205
class LoggingDecorator(ExtendedToOriginalDecorator):
1220
1206
def startTestRun(self):
1221
tests.ForwardingResult.startTestRun(self)
1207
ExtendedToOriginalDecorator.startTestRun(self)
1222
1208
calls.append('startTestRun')
1223
1209
test = unittest.FunctionTestCase(lambda:None)
1224
1210
stream = StringIO()
1655
1658
self.assertEqual('original', obj.test_attr)
1661
class _MissingFeature(tests.Feature):
1664
missing_feature = _MissingFeature()
1667
def _get_test(name):
1668
"""Get an instance of a specific example test.
1670
We protect this in a function so that they don't auto-run in the test
1674
class ExampleTests(tests.TestCase):
1676
def test_fail(self):
1677
mutter('this was a failing test')
1678
self.fail('this test will fail')
1680
def test_error(self):
1681
mutter('this test errored')
1682
raise RuntimeError('gotcha')
1684
def test_missing_feature(self):
1685
mutter('missing the feature')
1686
self.requireFeature(missing_feature)
1688
def test_skip(self):
1689
mutter('this test will be skipped')
1690
raise tests.TestSkipped('reason')
1692
def test_success(self):
1693
mutter('this test succeeds')
1695
def test_xfail(self):
1696
mutter('test with expected failure')
1697
self.knownFailure('this_fails')
1699
def test_unexpected_success(self):
1700
mutter('test with unexpected success')
1701
self.expectFailure('should_fail', lambda: None)
1703
return ExampleTests(name)
1706
class TestTestCaseLogDetails(tests.TestCase):
1708
def _run_test(self, test_name):
1709
test = _get_test(test_name)
1710
result = testtools.TestResult()
1714
def test_fail_has_log(self):
1715
result = self._run_test('test_fail')
1716
self.assertEqual(1, len(result.failures))
1717
result_content = result.failures[0][1]
1718
self.assertContainsRe(result_content, 'Text attachment: log')
1719
self.assertContainsRe(result_content, 'this was a failing test')
1721
def test_error_has_log(self):
1722
result = self._run_test('test_error')
1723
self.assertEqual(1, len(result.errors))
1724
result_content = result.errors[0][1]
1725
self.assertContainsRe(result_content, 'Text attachment: log')
1726
self.assertContainsRe(result_content, 'this test errored')
1728
def test_skip_has_no_log(self):
1729
result = self._run_test('test_skip')
1730
self.assertEqual(['reason'], result.skip_reasons.keys())
1731
skips = result.skip_reasons['reason']
1732
self.assertEqual(1, len(skips))
1734
self.assertFalse('log' in test.getDetails())
1736
def test_missing_feature_has_no_log(self):
1737
# testtools doesn't know about addNotSupported, so it just gets
1738
# considered as a skip
1739
result = self._run_test('test_missing_feature')
1740
self.assertEqual([missing_feature], result.skip_reasons.keys())
1741
skips = result.skip_reasons[missing_feature]
1742
self.assertEqual(1, len(skips))
1744
self.assertFalse('log' in test.getDetails())
1746
def test_xfail_has_no_log(self):
1747
result = self._run_test('test_xfail')
1748
self.assertEqual(1, len(result.expectedFailures))
1749
result_content = result.expectedFailures[0][1]
1750
self.assertNotContainsRe(result_content, 'Text attachment: log')
1751
self.assertNotContainsRe(result_content, 'test with expected failure')
1753
def test_unexpected_success_has_log(self):
1754
result = self._run_test('test_unexpected_success')
1755
self.assertEqual(1, len(result.unexpectedSuccesses))
1756
# Inconsistency, unexpectedSuccesses is a list of tests,
1757
# expectedFailures is a list of reasons?
1758
test = result.unexpectedSuccesses[0]
1759
details = test.getDetails()
1760
self.assertTrue('log' in details)
1763
class TestTestCloning(tests.TestCase):
1764
"""Tests that test cloning of TestCases (as used by multiply_tests)."""
1766
def test_cloned_testcase_does_not_share_details(self):
1767
"""A TestCase cloned with clone_test does not share mutable attributes
1768
such as details or cleanups.
1770
class Test(tests.TestCase):
1772
self.addDetail('foo', Content('text/plain', lambda: 'foo'))
1773
orig_test = Test('test_foo')
1774
cloned_test = tests.clone_test(orig_test, orig_test.id() + '(cloned)')
1775
orig_test.run(unittest.TestResult())
1776
self.assertEqual('foo', orig_test.getDetails()['foo'].iter_bytes())
1777
self.assertEqual(None, cloned_test.getDetails().get('foo'))
1779
def test_double_apply_scenario_preserves_first_scenario(self):
1780
"""Applying two levels of scenarios to a test preserves the attributes
1781
added by both scenarios.
1783
class Test(tests.TestCase):
1786
test = Test('test_foo')
1787
scenarios_x = [('x=1', {'x': 1}), ('x=2', {'x': 2})]
1788
scenarios_y = [('y=1', {'y': 1}), ('y=2', {'y': 2})]
1789
suite = tests.multiply_tests(test, scenarios_x, unittest.TestSuite())
1790
suite = tests.multiply_tests(suite, scenarios_y, unittest.TestSuite())
1791
all_tests = list(tests.iter_suite_tests(suite))
1792
self.assertLength(4, all_tests)
1793
all_xys = sorted((t.x, t.y) for t in all_tests)
1794
self.assertEqual([(1, 1), (1, 2), (2, 1), (2, 2)], all_xys)
1658
1797
# NB: Don't delete this; it's not actually from 0.11!
1659
1798
@deprecated_function(deprecated_in((0, 11, 0)))
1660
1799
def sample_deprecated_function():
1971
2107
load_list='missing file name', list_only=True)
2110
class TestSubunitLogDetails(tests.TestCase, SelfTestHelper):
2112
_test_needs_features = [features.subunit]
2114
def run_subunit_stream(self, test_name):
2115
from subunit import ProtocolTestCase
2117
return TestUtil.TestSuite([_get_test(test_name)])
2118
stream = self.run_selftest(runner_class=tests.SubUnitBzrRunner,
2119
test_suite_factory=factory)
2120
test = ProtocolTestCase(stream)
2121
result = testtools.TestResult()
2123
content = stream.getvalue()
2124
return content, result
2126
def test_fail_has_log(self):
2127
content, result = self.run_subunit_stream('test_fail')
2128
self.assertEqual(1, len(result.failures))
2129
self.assertContainsRe(content, '(?m)^log$')
2130
self.assertContainsRe(content, 'this test will fail')
2132
def test_error_has_log(self):
2133
content, result = self.run_subunit_stream('test_error')
2134
self.assertContainsRe(content, '(?m)^log$')
2135
self.assertContainsRe(content, 'this test errored')
2137
def test_skip_has_no_log(self):
2138
content, result = self.run_subunit_stream('test_skip')
2139
self.assertNotContainsRe(content, '(?m)^log$')
2140
self.assertNotContainsRe(content, 'this test will be skipped')
2141
self.assertEqual(['reason'], result.skip_reasons.keys())
2142
skips = result.skip_reasons['reason']
2143
self.assertEqual(1, len(skips))
2145
# RemotedTestCase doesn't preserve the "details"
2146
## self.assertFalse('log' in test.getDetails())
2148
def test_missing_feature_has_no_log(self):
2149
content, result = self.run_subunit_stream('test_missing_feature')
2150
self.assertNotContainsRe(content, '(?m)^log$')
2151
self.assertNotContainsRe(content, 'missing the feature')
2152
self.assertEqual(['_MissingFeature\n'], result.skip_reasons.keys())
2153
skips = result.skip_reasons['_MissingFeature\n']
2154
self.assertEqual(1, len(skips))
2156
# RemotedTestCase doesn't preserve the "details"
2157
## self.assertFalse('log' in test.getDetails())
2159
def test_xfail_has_no_log(self):
2160
content, result = self.run_subunit_stream('test_xfail')
2161
self.assertNotContainsRe(content, '(?m)^log$')
2162
self.assertNotContainsRe(content, 'test with expected failure')
2163
self.assertEqual(1, len(result.expectedFailures))
2164
result_content = result.expectedFailures[0][1]
2165
self.assertNotContainsRe(result_content, 'Text attachment: log')
2166
self.assertNotContainsRe(result_content, 'test with expected failure')
2168
def test_unexpected_success_has_log(self):
2169
content, result = self.run_subunit_stream('test_unexpected_success')
2170
self.assertContainsRe(content, '(?m)^log$')
2171
self.assertContainsRe(content, 'test with unexpected success')
2172
self.expectFailure('subunit treats "unexpectedSuccess"'
2173
' as a plain success',
2174
self.assertEqual, 1, len(result.unexpectedSuccesses))
2175
self.assertEqual(1, len(result.unexpectedSuccesses))
2176
test = result.unexpectedSuccesses[0]
2177
# RemotedTestCase doesn't preserve the "details"
2178
## self.assertTrue('log' in test.getDetails())
2180
def test_success_has_no_log(self):
2181
content, result = self.run_subunit_stream('test_success')
2182
self.assertEqual(1, result.testsRun)
2183
self.assertNotContainsRe(content, '(?m)^log$')
2184
self.assertNotContainsRe(content, 'this test succeeds')
1974
2187
class TestRunBzr(tests.TestCase):
2960
3179
self.assertEquals('bzrlib.plugins', tpr.resolve_alias('bp'))
3182
class TestThreadLeakDetection(tests.TestCase):
3183
"""Ensure when tests leak threads we detect and report it"""
3185
class LeakRecordingResult(tests.ExtendedTestResult):
3187
tests.ExtendedTestResult.__init__(self, StringIO(), 0, 1)
3189
def _report_thread_leak(self, test, leaks, alive):
3190
self.leaks.append((test, leaks))
3192
def test_testcase_without_addCleanups(self):
3193
"""Check old TestCase instances don't break with leak detection"""
3194
class Test(unittest.TestCase):
3197
result = self.LeakRecordingResult()
3199
result.startTestRun()
3201
result.stopTestRun()
3202
self.assertEqual(result._tests_leaking_threads_count, 0)
3203
self.assertEqual(result.leaks, [])
3205
def test_thread_leak(self):
3206
"""Ensure a thread that outlives the running of a test is reported
3208
Uses a thread that blocks on an event, and is started by the inner
3209
test case. As the thread outlives the inner case's run, it should be
3210
detected as a leak, but the event is then set so that the thread can
3211
be safely joined in cleanup so it's not leaked for real.
3213
event = threading.Event()
3214
thread = threading.Thread(name="Leaker", target=event.wait)
3215
class Test(tests.TestCase):
3216
def test_leak(self):
3218
result = self.LeakRecordingResult()
3219
test = Test("test_leak")
3220
self.addCleanup(thread.join)
3221
self.addCleanup(event.set)
3222
result.startTestRun()
3224
result.stopTestRun()
3225
self.assertEqual(result._tests_leaking_threads_count, 1)
3226
self.assertEqual(result._first_thread_leaker_id, test.id())
3227
self.assertEqual(result.leaks, [(test, set([thread]))])
3228
self.assertContainsString(result.stream.getvalue(), "leaking threads")
3230
def test_multiple_leaks(self):
3231
"""Check multiple leaks are blamed on the test cases at fault
3233
Same concept as the previous test, but has one inner test method that
3234
leaks two threads, and one that doesn't leak at all.
3236
event = threading.Event()
3237
thread_a = threading.Thread(name="LeakerA", target=event.wait)
3238
thread_b = threading.Thread(name="LeakerB", target=event.wait)
3239
thread_c = threading.Thread(name="LeakerC", target=event.wait)
3240
class Test(tests.TestCase):
3241
def test_first_leak(self):
3243
def test_second_no_leak(self):
3245
def test_third_leak(self):
3248
result = self.LeakRecordingResult()
3249
first_test = Test("test_first_leak")
3250
third_test = Test("test_third_leak")
3251
self.addCleanup(thread_a.join)
3252
self.addCleanup(thread_b.join)
3253
self.addCleanup(thread_c.join)
3254
self.addCleanup(event.set)
3255
result.startTestRun()
3257
[first_test, Test("test_second_no_leak"), third_test]
3259
result.stopTestRun()
3260
self.assertEqual(result._tests_leaking_threads_count, 2)
3261
self.assertEqual(result._first_thread_leaker_id, first_test.id())
3262
self.assertEqual(result.leaks, [
3263
(first_test, set([thread_b])),
3264
(third_test, set([thread_a, thread_c]))])
3265
self.assertContainsString(result.stream.getvalue(), "leaking threads")
3268
class TestPostMortemDebugging(tests.TestCase):
3269
"""Check post mortem debugging works when tests fail or error"""
3271
class TracebackRecordingResult(tests.ExtendedTestResult):
3273
tests.ExtendedTestResult.__init__(self, StringIO(), 0, 1)
3274
self.postcode = None
3275
def _post_mortem(self, tb=None):
3276
"""Record the code object at the end of the current traceback"""
3277
tb = tb or sys.exc_info()[2]
3280
while next is not None:
3283
self.postcode = tb.tb_frame.f_code
3284
def report_error(self, test, err):
3286
def report_failure(self, test, err):
3289
def test_location_unittest_error(self):
3290
"""Needs right post mortem traceback with erroring unittest case"""
3291
class Test(unittest.TestCase):
3294
result = self.TracebackRecordingResult()
3296
self.assertEqual(result.postcode, Test.runTest.func_code)
3298
def test_location_unittest_failure(self):
3299
"""Needs right post mortem traceback with failing unittest case"""
3300
class Test(unittest.TestCase):
3302
raise self.failureException
3303
result = self.TracebackRecordingResult()
3305
self.assertEqual(result.postcode, Test.runTest.func_code)
3307
def test_location_bt_error(self):
3308
"""Needs right post mortem traceback with erroring bzrlib.tests case"""
3309
class Test(tests.TestCase):
3310
def test_error(self):
3312
result = self.TracebackRecordingResult()
3313
Test("test_error").run(result)
3314
self.assertEqual(result.postcode, Test.test_error.func_code)
3316
def test_location_bt_failure(self):
3317
"""Needs right post mortem traceback with failing bzrlib.tests case"""
3318
class Test(tests.TestCase):
3319
def test_failure(self):
3320
raise self.failureException
3321
result = self.TracebackRecordingResult()
3322
Test("test_failure").run(result)
3323
self.assertEqual(result.postcode, Test.test_failure.func_code)
3325
def test_env_var_triggers_post_mortem(self):
3326
"""Check pdb.post_mortem is called iff BZR_TEST_PDB is set"""
3328
result = tests.ExtendedTestResult(StringIO(), 0, 1)
3329
post_mortem_calls = []
3330
self.overrideAttr(pdb, "post_mortem", post_mortem_calls.append)
3331
self.overrideEnv('BZR_TEST_PDB', None)
3332
result._post_mortem(1)
3333
self.overrideEnv('BZR_TEST_PDB', 'on')
3334
result._post_mortem(2)
3335
self.assertEqual([2], post_mortem_calls)
2963
3338
class TestRunSuite(tests.TestCase):
2965
3340
def test_runner_class(self):
2976
3351
self.verbosity)
2977
3352
tests.run_suite(suite, runner_class=MyRunner, stream=StringIO())
2978
3353
self.assertLength(1, calls)
3356
class TestEnvironHandling(tests.TestCase):
3358
def test_overrideEnv_None_called_twice_doesnt_leak(self):
3359
self.failIf('MYVAR' in os.environ)
3360
self.overrideEnv('MYVAR', '42')
3361
# We use an embedded test to make sure we fix the _captureVar bug
3362
class Test(tests.TestCase):
3364
# The first call save the 42 value
3365
self.overrideEnv('MYVAR', None)
3366
self.assertEquals(None, os.environ.get('MYVAR'))
3367
# Make sure we can call it twice
3368
self.overrideEnv('MYVAR', None)
3369
self.assertEquals(None, os.environ.get('MYVAR'))
3371
result = tests.TextTestResult(output, 0, 1)
3372
Test('test_me').run(result)
3373
if not result.wasStrictlySuccessful():
3374
self.fail(output.getvalue())
3375
# We get our value back
3376
self.assertEquals('42', os.environ.get('MYVAR'))
3379
class TestIsolatedEnv(tests.TestCase):
3380
"""Test isolating tests from os.environ.
3382
Since we use tests that are already isolated from os.environ a bit of care
3383
should be taken when designing the tests to avoid bootstrap side-effects.
3384
The tests start an already clean os.environ which allow doing valid
3385
assertions about which variables are present or not and design tests around
3389
class ScratchMonkey(tests.TestCase):
3394
def test_basics(self):
3395
# Make sure we know the definition of BZR_HOME: not part of os.environ
3396
# for tests.TestCase.
3397
self.assertTrue('BZR_HOME' in tests.isolated_environ)
3398
self.assertEquals(None, tests.isolated_environ['BZR_HOME'])
3399
# Being part of isolated_environ, BZR_HOME should not appear here
3400
self.assertFalse('BZR_HOME' in os.environ)
3401
# Make sure we know the definition of LINES: part of os.environ for
3403
self.assertTrue('LINES' in tests.isolated_environ)
3404
self.assertEquals('25', tests.isolated_environ['LINES'])
3405
self.assertEquals('25', os.environ['LINES'])
3407
def test_injecting_unknown_variable(self):
3408
# BZR_HOME is known to be absent from os.environ
3409
test = self.ScratchMonkey('test_me')
3410
tests.override_os_environ(test, {'BZR_HOME': 'foo'})
3411
self.assertEquals('foo', os.environ['BZR_HOME'])
3412
tests.restore_os_environ(test)
3413
self.assertFalse('BZR_HOME' in os.environ)
3415
def test_injecting_known_variable(self):
3416
test = self.ScratchMonkey('test_me')
3417
# LINES is known to be present in os.environ
3418
tests.override_os_environ(test, {'LINES': '42'})
3419
self.assertEquals('42', os.environ['LINES'])
3420
tests.restore_os_environ(test)
3421
self.assertEquals('25', os.environ['LINES'])
3423
def test_deleting_variable(self):
3424
test = self.ScratchMonkey('test_me')
3425
# LINES is known to be present in os.environ
3426
tests.override_os_environ(test, {'LINES': None})
3427
self.assertTrue('LINES' not in os.environ)
3428
tests.restore_os_environ(test)
3429
self.assertEquals('25', os.environ['LINES'])
3432
class TestDocTestSuiteIsolation(tests.TestCase):
3433
"""Test that `tests.DocTestSuite` isolates doc tests from os.environ.
3435
Since tests.TestCase alreay provides an isolation from os.environ, we use
3436
the clean environment as a base for testing. To precisely capture the
3437
isolation provided by tests.DocTestSuite, we use doctest.DocTestSuite to
3440
We want to make sure `tests.DocTestSuite` respect `tests.isolated_environ`,
3441
not `os.environ` so each test overrides it to suit its needs.
3445
def get_doctest_suite_for_string(self, klass, string):
3446
class Finder(doctest.DocTestFinder):
3448
def find(*args, **kwargs):
3449
test = doctest.DocTestParser().get_doctest(
3450
string, {}, 'foo', 'foo.py', 0)
3453
suite = klass(test_finder=Finder())
3456
def run_doctest_suite_for_string(self, klass, string):
3457
suite = self.get_doctest_suite_for_string(klass, string)
3459
result = tests.TextTestResult(output, 0, 1)
3461
return result, output
3463
def assertDocTestStringSucceds(self, klass, string):
3464
result, output = self.run_doctest_suite_for_string(klass, string)
3465
if not result.wasStrictlySuccessful():
3466
self.fail(output.getvalue())
3468
def assertDocTestStringFails(self, klass, string):
3469
result, output = self.run_doctest_suite_for_string(klass, string)
3470
if result.wasStrictlySuccessful():
3471
self.fail(output.getvalue())
3473
def test_injected_variable(self):
3474
self.overrideAttr(tests, 'isolated_environ', {'LINES': '42'})
3477
>>> os.environ['LINES']
3480
# doctest.DocTestSuite fails as it sees '25'
3481
self.assertDocTestStringFails(doctest.DocTestSuite, test)
3482
# tests.DocTestSuite sees '42'
3483
self.assertDocTestStringSucceds(tests.IsolatedDocTestSuite, test)
3485
def test_deleted_variable(self):
3486
self.overrideAttr(tests, 'isolated_environ', {'LINES': None})
3489
>>> os.environ.get('LINES')
3491
# doctest.DocTestSuite fails as it sees '25'
3492
self.assertDocTestStringFails(doctest.DocTestSuite, test)
3493
# tests.DocTestSuite sees None
3494
self.assertDocTestStringSucceds(tests.IsolatedDocTestSuite, test)