From 74abdd5eeb99c4ae212047504bed4106f3b09893 Mon Sep 17 00:00:00 2001 From: Michelle Tran Date: Wed, 23 Oct 2024 12:38:01 -0400 Subject: [PATCH] Fix leaky use of time machine The time sometimes affected other tests (they wouldn't reset back). --- tasks/tests/unit/test_process_flakes.py | 83 +++--- .../unit/test_test_results_processor_task.py | 275 +++++++++--------- 2 files changed, 177 insertions(+), 181 deletions(-) diff --git a/tasks/tests/unit/test_process_flakes.py b/tasks/tests/unit/test_process_flakes.py index 181a13123..d7d174438 100644 --- a/tasks/tests/unit/test_process_flakes.py +++ b/tasks/tests/unit/test_process_flakes.py @@ -460,57 +460,50 @@ def test_it_creates_flakes_expires(): @pytest.mark.django_db def test_it_creates_rollups(): - traveller = time_machine.travel("1970-1-1T00:00:00Z") - traveller.start() - rs = RepoSimulator() - commits = [] - c1 = rs.create_commit() - rs.merge(c1) - rs.add_test_instance(c1, outcome=TestInstance.Outcome.FAILURE.value) - rs.add_test_instance(c1, outcome=TestInstance.Outcome.FAILURE.value) - - ProcessFlakesTask().run_impl( - None, - repo_id=rs.repo.repoid, - commit_id_list=[c1.commitid], - branch=rs.repo.branch, - ) - - traveller.stop() - - traveller = time_machine.travel("1970-1-2T00:00:00Z") - traveller.start() + with time_machine.travel("1970-1-1T00:00:00Z"): + rs = RepoSimulator() + commits = [] + c1 = rs.create_commit() + rs.merge(c1) + rs.add_test_instance(c1, outcome=TestInstance.Outcome.FAILURE.value) + rs.add_test_instance(c1, outcome=TestInstance.Outcome.FAILURE.value) - c2 = rs.create_commit() - rs.merge(c2) - rs.add_test_instance(c2, outcome=TestInstance.Outcome.FAILURE.value) - rs.add_test_instance(c2, outcome=TestInstance.Outcome.FAILURE.value) + ProcessFlakesTask().run_impl( + None, + repo_id=rs.repo.repoid, + commit_id_list=[c1.commitid], + branch=rs.repo.branch, + ) - ProcessFlakesTask().run_impl( - None, - repo_id=rs.repo.repoid, - commit_id_list=[c2.commitid], - branch=rs.repo.branch, - ) + with time_machine.travel("1970-1-2T00:00:00Z"): + c2 = rs.create_commit() + rs.merge(c2) + rs.add_test_instance(c2, outcome=TestInstance.Outcome.FAILURE.value) + rs.add_test_instance(c2, outcome=TestInstance.Outcome.FAILURE.value) - rollups = DailyTestRollup.objects.all() + ProcessFlakesTask().run_impl( + None, + repo_id=rs.repo.repoid, + commit_id_list=[c2.commitid], + branch=rs.repo.branch, + ) - assert len(rollups) == 4 + rollups = DailyTestRollup.objects.all() - assert rollups[0].fail_count == 1 - assert rollups[0].flaky_fail_count == 1 - assert rollups[0].date == dt.date.today() - dt.timedelta(days=1) + assert len(rollups) == 4 - assert rollups[1].fail_count == 1 - assert rollups[1].flaky_fail_count == 1 - assert rollups[1].date == dt.date.today() - dt.timedelta(days=1) + assert rollups[0].fail_count == 1 + assert rollups[0].flaky_fail_count == 1 + assert rollups[0].date == dt.date.today() - dt.timedelta(days=1) - assert rollups[2].fail_count == 1 - assert rollups[2].flaky_fail_count == 1 - assert rollups[2].date == dt.date.today() + assert rollups[1].fail_count == 1 + assert rollups[1].flaky_fail_count == 1 + assert rollups[1].date == dt.date.today() - dt.timedelta(days=1) - assert rollups[3].fail_count == 1 - assert rollups[3].flaky_fail_count == 1 - assert rollups[3].date == dt.date.today() + assert rollups[2].fail_count == 1 + assert rollups[2].flaky_fail_count == 1 + assert rollups[2].date == dt.date.today() - traveller.stop() + assert rollups[3].fail_count == 1 + assert rollups[3].flaky_fail_count == 1 + assert rollups[3].date == dt.date.today() diff --git a/tasks/tests/unit/test_test_results_processor_task.py b/tasks/tests/unit/test_test_results_processor_task.py index 4260b0222..844295b59 100644 --- a/tasks/tests/unit/test_test_results_processor_task.py +++ b/tasks/tests/unit/test_test_results_processor_task.py @@ -488,152 +488,155 @@ def test_upload_processor_task_call_daily_test_totals( mock_redis, celery_app, ): - traveller = travel("1970-1-1T00:00:00Z", tick=False) - traveller.start() - first_url = "v4/raw/2019-05-22/C3C4715CA57C910D11D5EB899FC86A7E/4c4e4654ac25037ae869caeb3619d485970b6304/a84d445c-9c1e-434f-8275-f18f1f320f81.txt" - with open( - here.parent.parent / "samples" / "sample_multi_test_part_1.json" - ) as f: - content = f.read() - mock_storage.write_file("archive", first_url, content) - - first_commit = CommitFactory.create( - message="hello world", - commitid="cd76b0821854a780b60012aed85af0a8263004ad", - repository__owner__unencrypted_oauth_token="test7lk5ndmtqzxlx06rip65nac9c7epqopclnoy", - repository__owner__username="joseph-sentry", - repository__owner__service="github", - repository__name="codecov-demo", - branch="first_branch", - ) - dbsession.add(first_commit) - dbsession.flush() - - first_report_row = CommitReport(commit_id=first_commit.id_) - dbsession.add(first_report_row) - dbsession.flush() - - upload = UploadFactory.create(storage_path=first_url, report=first_report_row) - dbsession.add(upload) - dbsession.flush() - - repoid = upload.report.commit.repoid - redis_queue = [{"url": first_url, "upload_pk": upload.id_}] - mocker.patch.object(TestResultsProcessorTask, "app", celery_app) - - result = TestResultsProcessorTask().run_impl( - dbsession, - repoid=repoid, - commitid=first_commit.commitid, - commit_yaml={"codecov": {"max_report_age": False}}, - arguments_list=redis_queue, - ) - expected_result = [ - { - "successful": True, - } - ] - - rollups = dbsession.query(DailyTestRollup).all() - - assert [r.branch for r in rollups] == [ - "first_branch", - "first_branch", - ] - - assert [r.date for r in rollups] == [ - date.today(), - date.today(), - ] - - traveller.stop() - - traveller = travel("1970-1-2T00:00:00Z", tick=False) - traveller.start() - - second_commit = CommitFactory.create( - message="hello world 2", - commitid="bd76b0821854a780b60012aed85af0a8263004ad", - repository=first_commit.repository, - branch="second_branch", - ) - dbsession.add(second_commit) - dbsession.flush() - - second_report_row = CommitReport(commit_id=second_commit.id_) - dbsession.add(second_report_row) - dbsession.flush() - - second_url = "v4/raw/2019-05-22/C3C4715CA57C910D11D5EB899FC86A7E/4c4e4654ac25037ae869caeb3619d485970b6304/b84d445c-9c1e-434f-8275-f18f1f320f81.txt" - with open( - here.parent.parent / "samples" / "sample_multi_test_part_2.json" - ) as f: - content = f.read() - mock_storage.write_file("archive", second_url, content) - upload = UploadFactory.create(storage_path=second_url, report=second_report_row) - dbsession.add(upload) - dbsession.flush() - - tests = dbsession.query(Test).all() - for test in tests: - flake = FlakeFactory.create(test=test) - dbsession.add(flake) + with travel("1970-1-1T00:00:00Z", tick=False): + first_url = "v4/raw/2019-05-22/C3C4715CA57C910D11D5EB899FC86A7E/4c4e4654ac25037ae869caeb3619d485970b6304/a84d445c-9c1e-434f-8275-f18f1f320f81.txt" + with open( + here.parent.parent / "samples" / "sample_multi_test_part_1.json" + ) as f: + content = f.read() + mock_storage.write_file("archive", first_url, content) + + first_commit = CommitFactory.create( + message="hello world", + commitid="cd76b0821854a780b60012aed85af0a8263004ad", + repository__owner__unencrypted_oauth_token="test7lk5ndmtqzxlx06rip65nac9c7epqopclnoy", + repository__owner__username="joseph-sentry", + repository__owner__service="github", + repository__name="codecov-demo", + branch="first_branch", + ) + dbsession.add(first_commit) dbsession.flush() - redis_queue = [{"url": second_url, "upload_pk": upload.id_}] + first_report_row = CommitReport(commit_id=first_commit.id_) + dbsession.add(first_report_row) + dbsession.flush() - result = TestResultsProcessorTask().run_impl( - dbsession, - repoid=repoid, - commitid=second_commit.commitid, - commit_yaml={"codecov": {"max_report_age": False}}, - arguments_list=redis_queue, - ) - expected_result = [ - { - "successful": True, - } - ] + upload = UploadFactory.create( + storage_path=first_url, report=first_report_row + ) + dbsession.add(upload) + dbsession.flush() - rollups: list[DailyTestRollup] = dbsession.query(DailyTestRollup).all() + repoid = upload.report.commit.repoid + redis_queue = [{"url": first_url, "upload_pk": upload.id_}] + mocker.patch.object(TestResultsProcessorTask, "app", celery_app) - assert result == expected_result + result = TestResultsProcessorTask().run_impl( + dbsession, + repoid=repoid, + commitid=first_commit.commitid, + commit_yaml={"codecov": {"max_report_age": False}}, + arguments_list=redis_queue, + ) + expected_result = [ + { + "successful": True, + } + ] + + rollups = dbsession.query(DailyTestRollup).all() + + assert [r.branch for r in rollups] == [ + "first_branch", + "first_branch", + ] + + assert [r.date for r in rollups] == [ + date.today(), + date.today(), + ] + + with travel("1970-1-2T00:00:00Z", tick=False): + second_commit = CommitFactory.create( + message="hello world 2", + commitid="bd76b0821854a780b60012aed85af0a8263004ad", + repository=first_commit.repository, + branch="second_branch", + ) + dbsession.add(second_commit) + dbsession.flush() - assert [r.branch for r in rollups] == [ - "first_branch", - "first_branch", - "second_branch", - "second_branch", - ] + second_report_row = CommitReport(commit_id=second_commit.id_) + dbsession.add(second_report_row) + dbsession.flush() - assert [r.date for r in rollups] == [ - date.today() - timedelta(days=1), - date.today() - timedelta(days=1), - date.today(), - date.today(), - ] + second_url = "v4/raw/2019-05-22/C3C4715CA57C910D11D5EB899FC86A7E/4c4e4654ac25037ae869caeb3619d485970b6304/b84d445c-9c1e-434f-8275-f18f1f320f81.txt" + with open( + here.parent.parent / "samples" / "sample_multi_test_part_2.json" + ) as f: + content = f.read() + mock_storage.write_file("archive", second_url, content) + upload = UploadFactory.create( + storage_path=second_url, report=second_report_row + ) + dbsession.add(upload) + dbsession.flush() - assert [r.fail_count for r in rollups] == [1, 0, 0, 1] - assert [r.pass_count for r in rollups] == [1, 1, 2, 0] - assert [r.skip_count for r in rollups] == [0, 0, 0, 0] - assert [r.flaky_fail_count for r in rollups] == [0, 0, 0, 1] + tests = dbsession.query(Test).all() + for test in tests: + flake = FlakeFactory.create(test=test) + dbsession.add(flake) + dbsession.flush() - assert [r.commits_where_fail for r in rollups] == [ - ["cd76b0821854a780b60012aed85af0a8263004ad"], - [], - [], - ["bd76b0821854a780b60012aed85af0a8263004ad"], - ] + redis_queue = [{"url": second_url, "upload_pk": upload.id_}] - assert [r.latest_run for r in rollups] == [ - datetime(1970, 1, 1, 0, 0, tzinfo=timezone.utc), - datetime(1970, 1, 1, 0, 0, tzinfo=timezone.utc), - datetime(1970, 1, 2, 0, 0, tzinfo=timezone.utc), - datetime(1970, 1, 2, 0, 0, tzinfo=timezone.utc), - ] - assert [r.avg_duration_seconds for r in rollups] == [0.001, 7.2, 0.002, 3.6] - assert [r.last_duration_seconds for r in rollups] == [0.001, 7.2, 0.002, 3.6] - traveller.stop() + result = TestResultsProcessorTask().run_impl( + dbsession, + repoid=repoid, + commitid=second_commit.commitid, + commit_yaml={"codecov": {"max_report_age": False}}, + arguments_list=redis_queue, + ) + expected_result = [ + { + "successful": True, + } + ] + + rollups: list[DailyTestRollup] = dbsession.query(DailyTestRollup).all() + + assert result == expected_result + + assert [r.branch for r in rollups] == [ + "first_branch", + "first_branch", + "second_branch", + "second_branch", + ] + + assert [r.date for r in rollups] == [ + date.today() - timedelta(days=1), + date.today() - timedelta(days=1), + date.today(), + date.today(), + ] + + assert [r.fail_count for r in rollups] == [1, 0, 0, 1] + assert [r.pass_count for r in rollups] == [1, 1, 2, 0] + assert [r.skip_count for r in rollups] == [0, 0, 0, 0] + assert [r.flaky_fail_count for r in rollups] == [0, 0, 0, 1] + + assert [r.commits_where_fail for r in rollups] == [ + ["cd76b0821854a780b60012aed85af0a8263004ad"], + [], + [], + ["bd76b0821854a780b60012aed85af0a8263004ad"], + ] + + assert [r.latest_run for r in rollups] == [ + datetime(1970, 1, 1, 0, 0, tzinfo=timezone.utc), + datetime(1970, 1, 1, 0, 0, tzinfo=timezone.utc), + datetime(1970, 1, 2, 0, 0, tzinfo=timezone.utc), + datetime(1970, 1, 2, 0, 0, tzinfo=timezone.utc), + ] + assert [r.avg_duration_seconds for r in rollups] == [0.001, 7.2, 0.002, 3.6] + assert [r.last_duration_seconds for r in rollups] == [ + 0.001, + 7.2, + 0.002, + 3.6, + ] @pytest.mark.integration @pytest.mark.parametrize(