diff --git a/stestr/commands/failing.py b/stestr/commands/failing.py index 87818a01..9d4bb399 100644 --- a/stestr/commands/failing.py +++ b/stestr/commands/failing.py @@ -113,7 +113,7 @@ def failing(repo_type='file', repo_url=None, list_tests=False, subunit=False, case.run(result) finally: result.stopTestRun() - failed = not summary.wasSuccessful() + failed = not results.wasSuccessful(summary) if failed: result = 1 else: diff --git a/stestr/commands/last.py b/stestr/commands/last.py index 46f7ce9d..2fd2540c 100644 --- a/stestr/commands/last.py +++ b/stestr/commands/last.py @@ -154,7 +154,7 @@ def last(repo_type='file', repo_url=None, subunit_out=False, pretty_out=True, case.run(output_result) finally: output_result.stopTestRun() - failed = not summary.wasSuccessful() + failed = not results.wasSuccessful(summary) else: stream = latest_run.get_subunit_stream() failed = subunit_trace.trace(stream, stdout, post_fails=True, diff --git a/stestr/commands/load.py b/stestr/commands/load.py index 673b57c3..8d6304ba 100644 --- a/stestr/commands/load.py +++ b/stestr/commands/load.py @@ -219,7 +219,7 @@ def make_tests(): if pretty_out and not subunit_out: subunit_trace.print_fails(stdout) subunit_trace.print_summary(stdout, elapsed_time) - if not summary_result.wasSuccessful(): + if not results.wasSuccessful(summary_result): return 1 else: return 0 diff --git a/stestr/commands/run.py b/stestr/commands/run.py index 8a5df531..5266f6ae 100644 --- a/stestr/commands/run.py +++ b/stestr/commands/run.py @@ -29,6 +29,7 @@ from stestr import output from stestr.repository import abstract as repository from stestr.repository import util +from stestr import results from stestr.testlist import parse_list from stestr import user_config @@ -346,7 +347,7 @@ def run_tests(): stream.run(summary) finally: summary.stopTestRun() - if not summary.wasSuccessful(): + if not results.wasSuccessful(summary): result = 1 if result: return result @@ -495,7 +496,7 @@ def run_tests(): stream.run(summary) finally: summary.stopTestRun() - if not summary.wasSuccessful(): + if not results.wasSuccessful(summary): result = 1 if result: return result diff --git a/stestr/results.py b/stestr/results.py index 13e9d249..5e630271 100644 --- a/stestr/results.py +++ b/stestr/results.py @@ -17,6 +17,11 @@ from stestr import output +def wasSuccessful(summary): + return not (summary.errors or summary.failures or + summary.unexpectedSuccesses) + + class SummarizingResult(testtools.StreamSummary): def __init__(self): diff --git a/stestr/subunit_trace.py b/stestr/subunit_trace.py index 1461a859..a1f8a72c 100644 --- a/stestr/subunit_trace.py +++ b/stestr/subunit_trace.py @@ -29,6 +29,7 @@ import testtools from stestr import colorizer +from stestr import results # NOTE(mtreinish) on python3 anydbm was renamed dbm and the python2 dbm module # was renamed to dbm.ndbm, this block takes that into account @@ -388,7 +389,7 @@ def trace(stdin, stdout, print_failures=False, failonly=False, if count_tests('status', '^success$') == 0: print("\nNo tests were successful during the run") return 1 - return 0 if summary.wasSuccessful() else 1 + return 0 if results.wasSuccessful(summary) else 1 def main(): diff --git a/stestr/tests/files/failing-tests b/stestr/tests/files/failing-tests index 78efc938..5d9dbf0b 100644 --- a/stestr/tests/files/failing-tests +++ b/stestr/tests/files/failing-tests @@ -21,3 +21,7 @@ class FakeTestClass(testtools.TestCase): def test_pass_list(self): test_list = ['test', 'a', 'b'] self.assertIn('fail', test_list) + + def test_unexpected_pass(self): + self.expectFailure("we are sad", + self.assertEqual, 1, 1) diff --git a/stestr/tests/files/passing-tests b/stestr/tests/files/passing-tests index a55cb1b5..3d9b2728 100644 --- a/stestr/tests/files/passing-tests +++ b/stestr/tests/files/passing-tests @@ -21,3 +21,7 @@ class FakeTestClass(testtools.TestCase): def test_pass_list(self): test_list = ['test', 'a', 'b'] self.assertIn('test', test_list) + + def test_xfail(self): + self.expectFailure("we are sad", + self.assertEqual, 1, 0) diff --git a/stestr/tests/test_return_codes.py b/stestr/tests/test_return_codes.py index d4e9bafb..6ae99c22 100644 --- a/stestr/tests/test_return_codes.py +++ b/stestr/tests/test_return_codes.py @@ -127,6 +127,12 @@ def test_parallel_passing_bad_regex(self): def test_parallel_fails(self): self.assertRunExit('stestr run', 1) + def test_parallel_passing_xfail(self): + self.assertRunExit('stestr run xfail', 0) + + def test_parallel_fails_unxsuccess(self): + self.assertRunExit('stestr run unexpected', 1) + def test_parallel_blacklist(self): fd, path = tempfile.mkstemp() self.addCleanup(os.remove, path)