Skip to content

Commit

Permalink
Fix leaky use of time machine
Browse files Browse the repository at this point in the history
The time sometimes affected other tests (they wouldn't reset back).
  • Loading branch information
michelletran-codecov committed Oct 23, 2024
1 parent af27256 commit 74abdd5
Show file tree
Hide file tree
Showing 2 changed files with 177 additions and 181 deletions.
83 changes: 38 additions & 45 deletions tasks/tests/unit/test_process_flakes.py
Original file line number Diff line number Diff line change
Expand Up @@ -460,57 +460,50 @@ def test_it_creates_flakes_expires():

@pytest.mark.django_db
def test_it_creates_rollups():
traveller = time_machine.travel("1970-1-1T00:00:00Z")
traveller.start()
rs = RepoSimulator()
commits = []
c1 = rs.create_commit()
rs.merge(c1)
rs.add_test_instance(c1, outcome=TestInstance.Outcome.FAILURE.value)
rs.add_test_instance(c1, outcome=TestInstance.Outcome.FAILURE.value)

ProcessFlakesTask().run_impl(
None,
repo_id=rs.repo.repoid,
commit_id_list=[c1.commitid],
branch=rs.repo.branch,
)

traveller.stop()

traveller = time_machine.travel("1970-1-2T00:00:00Z")
traveller.start()
with time_machine.travel("1970-1-1T00:00:00Z"):
rs = RepoSimulator()
commits = []
c1 = rs.create_commit()
rs.merge(c1)
rs.add_test_instance(c1, outcome=TestInstance.Outcome.FAILURE.value)
rs.add_test_instance(c1, outcome=TestInstance.Outcome.FAILURE.value)

c2 = rs.create_commit()
rs.merge(c2)
rs.add_test_instance(c2, outcome=TestInstance.Outcome.FAILURE.value)
rs.add_test_instance(c2, outcome=TestInstance.Outcome.FAILURE.value)
ProcessFlakesTask().run_impl(
None,
repo_id=rs.repo.repoid,
commit_id_list=[c1.commitid],
branch=rs.repo.branch,
)

ProcessFlakesTask().run_impl(
None,
repo_id=rs.repo.repoid,
commit_id_list=[c2.commitid],
branch=rs.repo.branch,
)
with time_machine.travel("1970-1-2T00:00:00Z"):
c2 = rs.create_commit()
rs.merge(c2)
rs.add_test_instance(c2, outcome=TestInstance.Outcome.FAILURE.value)
rs.add_test_instance(c2, outcome=TestInstance.Outcome.FAILURE.value)

rollups = DailyTestRollup.objects.all()
ProcessFlakesTask().run_impl(
None,
repo_id=rs.repo.repoid,
commit_id_list=[c2.commitid],
branch=rs.repo.branch,
)

assert len(rollups) == 4
rollups = DailyTestRollup.objects.all()

assert rollups[0].fail_count == 1
assert rollups[0].flaky_fail_count == 1
assert rollups[0].date == dt.date.today() - dt.timedelta(days=1)
assert len(rollups) == 4

assert rollups[1].fail_count == 1
assert rollups[1].flaky_fail_count == 1
assert rollups[1].date == dt.date.today() - dt.timedelta(days=1)
assert rollups[0].fail_count == 1
assert rollups[0].flaky_fail_count == 1
assert rollups[0].date == dt.date.today() - dt.timedelta(days=1)

assert rollups[2].fail_count == 1
assert rollups[2].flaky_fail_count == 1
assert rollups[2].date == dt.date.today()
assert rollups[1].fail_count == 1
assert rollups[1].flaky_fail_count == 1
assert rollups[1].date == dt.date.today() - dt.timedelta(days=1)

Check warning on line 501 in tasks/tests/unit/test_process_flakes.py

View check run for this annotation

Codecov Notifications / codecov/patch

tasks/tests/unit/test_process_flakes.py#L499-L501

Added lines #L499 - L501 were not covered by tests

assert rollups[3].fail_count == 1
assert rollups[3].flaky_fail_count == 1
assert rollups[3].date == dt.date.today()
assert rollups[2].fail_count == 1
assert rollups[2].flaky_fail_count == 1
assert rollups[2].date == dt.date.today()

Check warning on line 505 in tasks/tests/unit/test_process_flakes.py

View check run for this annotation

Codecov Notifications / codecov/patch

tasks/tests/unit/test_process_flakes.py#L503-L505

Added lines #L503 - L505 were not covered by tests

traveller.stop()
assert rollups[3].fail_count == 1
assert rollups[3].flaky_fail_count == 1
assert rollups[3].date == dt.date.today()

Check warning on line 509 in tasks/tests/unit/test_process_flakes.py

View check run for this annotation

Codecov Notifications / codecov/patch

tasks/tests/unit/test_process_flakes.py#L507-L509

Added lines #L507 - L509 were not covered by tests
275 changes: 139 additions & 136 deletions tasks/tests/unit/test_test_results_processor_task.py
Original file line number Diff line number Diff line change
Expand Up @@ -488,152 +488,155 @@ def test_upload_processor_task_call_daily_test_totals(
mock_redis,
celery_app,
):
traveller = travel("1970-1-1T00:00:00Z", tick=False)
traveller.start()
first_url = "v4/raw/2019-05-22/C3C4715CA57C910D11D5EB899FC86A7E/4c4e4654ac25037ae869caeb3619d485970b6304/a84d445c-9c1e-434f-8275-f18f1f320f81.txt"
with open(
here.parent.parent / "samples" / "sample_multi_test_part_1.json"
) as f:
content = f.read()
mock_storage.write_file("archive", first_url, content)

first_commit = CommitFactory.create(
message="hello world",
commitid="cd76b0821854a780b60012aed85af0a8263004ad",
repository__owner__unencrypted_oauth_token="test7lk5ndmtqzxlx06rip65nac9c7epqopclnoy",
repository__owner__username="joseph-sentry",
repository__owner__service="github",
repository__name="codecov-demo",
branch="first_branch",
)
dbsession.add(first_commit)
dbsession.flush()

first_report_row = CommitReport(commit_id=first_commit.id_)
dbsession.add(first_report_row)
dbsession.flush()

upload = UploadFactory.create(storage_path=first_url, report=first_report_row)
dbsession.add(upload)
dbsession.flush()

repoid = upload.report.commit.repoid
redis_queue = [{"url": first_url, "upload_pk": upload.id_}]
mocker.patch.object(TestResultsProcessorTask, "app", celery_app)

result = TestResultsProcessorTask().run_impl(
dbsession,
repoid=repoid,
commitid=first_commit.commitid,
commit_yaml={"codecov": {"max_report_age": False}},
arguments_list=redis_queue,
)
expected_result = [
{
"successful": True,
}
]

rollups = dbsession.query(DailyTestRollup).all()

assert [r.branch for r in rollups] == [
"first_branch",
"first_branch",
]

assert [r.date for r in rollups] == [
date.today(),
date.today(),
]

traveller.stop()

traveller = travel("1970-1-2T00:00:00Z", tick=False)
traveller.start()

second_commit = CommitFactory.create(
message="hello world 2",
commitid="bd76b0821854a780b60012aed85af0a8263004ad",
repository=first_commit.repository,
branch="second_branch",
)
dbsession.add(second_commit)
dbsession.flush()

second_report_row = CommitReport(commit_id=second_commit.id_)
dbsession.add(second_report_row)
dbsession.flush()

second_url = "v4/raw/2019-05-22/C3C4715CA57C910D11D5EB899FC86A7E/4c4e4654ac25037ae869caeb3619d485970b6304/b84d445c-9c1e-434f-8275-f18f1f320f81.txt"
with open(
here.parent.parent / "samples" / "sample_multi_test_part_2.json"
) as f:
content = f.read()
mock_storage.write_file("archive", second_url, content)
upload = UploadFactory.create(storage_path=second_url, report=second_report_row)
dbsession.add(upload)
dbsession.flush()

tests = dbsession.query(Test).all()
for test in tests:
flake = FlakeFactory.create(test=test)
dbsession.add(flake)
with travel("1970-1-1T00:00:00Z", tick=False):
first_url = "v4/raw/2019-05-22/C3C4715CA57C910D11D5EB899FC86A7E/4c4e4654ac25037ae869caeb3619d485970b6304/a84d445c-9c1e-434f-8275-f18f1f320f81.txt"
with open(

Check warning on line 493 in tasks/tests/unit/test_test_results_processor_task.py

View check run for this annotation

Codecov Notifications / codecov/patch

tasks/tests/unit/test_test_results_processor_task.py#L491-L493

Added lines #L491 - L493 were not covered by tests
here.parent.parent / "samples" / "sample_multi_test_part_1.json"
) as f:
content = f.read()
mock_storage.write_file("archive", first_url, content)

Check warning on line 497 in tasks/tests/unit/test_test_results_processor_task.py

View check run for this annotation

Codecov Notifications / codecov/patch

tasks/tests/unit/test_test_results_processor_task.py#L496-L497

Added lines #L496 - L497 were not covered by tests

first_commit = CommitFactory.create(

Check warning on line 499 in tasks/tests/unit/test_test_results_processor_task.py

View check run for this annotation

Codecov Notifications / codecov/patch

tasks/tests/unit/test_test_results_processor_task.py#L499

Added line #L499 was not covered by tests
message="hello world",
commitid="cd76b0821854a780b60012aed85af0a8263004ad",
repository__owner__unencrypted_oauth_token="test7lk5ndmtqzxlx06rip65nac9c7epqopclnoy",
repository__owner__username="joseph-sentry",
repository__owner__service="github",
repository__name="codecov-demo",
branch="first_branch",
)
dbsession.add(first_commit)

Check warning on line 508 in tasks/tests/unit/test_test_results_processor_task.py

View check run for this annotation

Codecov Notifications / codecov/patch

tasks/tests/unit/test_test_results_processor_task.py#L508

Added line #L508 was not covered by tests
dbsession.flush()

redis_queue = [{"url": second_url, "upload_pk": upload.id_}]
first_report_row = CommitReport(commit_id=first_commit.id_)
dbsession.add(first_report_row)
dbsession.flush()

Check warning on line 513 in tasks/tests/unit/test_test_results_processor_task.py

View check run for this annotation

Codecov Notifications / codecov/patch

tasks/tests/unit/test_test_results_processor_task.py#L511-L513

Added lines #L511 - L513 were not covered by tests

result = TestResultsProcessorTask().run_impl(
dbsession,
repoid=repoid,
commitid=second_commit.commitid,
commit_yaml={"codecov": {"max_report_age": False}},
arguments_list=redis_queue,
)
expected_result = [
{
"successful": True,
}
]
upload = UploadFactory.create(

Check warning on line 515 in tasks/tests/unit/test_test_results_processor_task.py

View check run for this annotation

Codecov Notifications / codecov/patch

tasks/tests/unit/test_test_results_processor_task.py#L515

Added line #L515 was not covered by tests
storage_path=first_url, report=first_report_row
)
dbsession.add(upload)
dbsession.flush()

Check warning on line 519 in tasks/tests/unit/test_test_results_processor_task.py

View check run for this annotation

Codecov Notifications / codecov/patch

tasks/tests/unit/test_test_results_processor_task.py#L518-L519

Added lines #L518 - L519 were not covered by tests

rollups: list[DailyTestRollup] = dbsession.query(DailyTestRollup).all()
repoid = upload.report.commit.repoid
redis_queue = [{"url": first_url, "upload_pk": upload.id_}]
mocker.patch.object(TestResultsProcessorTask, "app", celery_app)

Check warning on line 523 in tasks/tests/unit/test_test_results_processor_task.py

View check run for this annotation

Codecov Notifications / codecov/patch

tasks/tests/unit/test_test_results_processor_task.py#L521-L523

Added lines #L521 - L523 were not covered by tests

assert result == expected_result
result = TestResultsProcessorTask().run_impl(

Check warning on line 525 in tasks/tests/unit/test_test_results_processor_task.py

View check run for this annotation

Codecov Notifications / codecov/patch

tasks/tests/unit/test_test_results_processor_task.py#L525

Added line #L525 was not covered by tests
dbsession,
repoid=repoid,
commitid=first_commit.commitid,
commit_yaml={"codecov": {"max_report_age": False}},
arguments_list=redis_queue,
)
expected_result = [

Check warning on line 532 in tasks/tests/unit/test_test_results_processor_task.py

View check run for this annotation

Codecov Notifications / codecov/patch

tasks/tests/unit/test_test_results_processor_task.py#L532

Added line #L532 was not covered by tests
{
"successful": True,
}
]

rollups = dbsession.query(DailyTestRollup).all()

Check warning on line 538 in tasks/tests/unit/test_test_results_processor_task.py

View check run for this annotation

Codecov Notifications / codecov/patch

tasks/tests/unit/test_test_results_processor_task.py#L538

Added line #L538 was not covered by tests

assert [r.branch for r in rollups] == [

Check warning on line 540 in tasks/tests/unit/test_test_results_processor_task.py

View check run for this annotation

Codecov Notifications / codecov/patch

tasks/tests/unit/test_test_results_processor_task.py#L540

Added line #L540 was not covered by tests
"first_branch",
"first_branch",
]

assert [r.date for r in rollups] == [

Check warning on line 545 in tasks/tests/unit/test_test_results_processor_task.py

View check run for this annotation

Codecov Notifications / codecov/patch

tasks/tests/unit/test_test_results_processor_task.py#L545

Added line #L545 was not covered by tests
date.today(),
date.today(),
]

with travel("1970-1-2T00:00:00Z", tick=False):
second_commit = CommitFactory.create(

Check warning on line 551 in tasks/tests/unit/test_test_results_processor_task.py

View check run for this annotation

Codecov Notifications / codecov/patch

tasks/tests/unit/test_test_results_processor_task.py#L550-L551

Added lines #L550 - L551 were not covered by tests
message="hello world 2",
commitid="bd76b0821854a780b60012aed85af0a8263004ad",
repository=first_commit.repository,
branch="second_branch",
)
dbsession.add(second_commit)
dbsession.flush()

Check warning on line 558 in tasks/tests/unit/test_test_results_processor_task.py

View check run for this annotation

Codecov Notifications / codecov/patch

tasks/tests/unit/test_test_results_processor_task.py#L557-L558

Added lines #L557 - L558 were not covered by tests

assert [r.branch for r in rollups] == [
"first_branch",
"first_branch",
"second_branch",
"second_branch",
]
second_report_row = CommitReport(commit_id=second_commit.id_)
dbsession.add(second_report_row)
dbsession.flush()

Check warning on line 562 in tasks/tests/unit/test_test_results_processor_task.py

View check run for this annotation

Codecov Notifications / codecov/patch

tasks/tests/unit/test_test_results_processor_task.py#L560-L562

Added lines #L560 - L562 were not covered by tests

assert [r.date for r in rollups] == [
date.today() - timedelta(days=1),
date.today() - timedelta(days=1),
date.today(),
date.today(),
]
second_url = "v4/raw/2019-05-22/C3C4715CA57C910D11D5EB899FC86A7E/4c4e4654ac25037ae869caeb3619d485970b6304/b84d445c-9c1e-434f-8275-f18f1f320f81.txt"
with open(

Check warning on line 565 in tasks/tests/unit/test_test_results_processor_task.py

View check run for this annotation

Codecov Notifications / codecov/patch

tasks/tests/unit/test_test_results_processor_task.py#L564-L565

Added lines #L564 - L565 were not covered by tests
here.parent.parent / "samples" / "sample_multi_test_part_2.json"
) as f:
content = f.read()
mock_storage.write_file("archive", second_url, content)
upload = UploadFactory.create(

Check warning on line 570 in tasks/tests/unit/test_test_results_processor_task.py

View check run for this annotation

Codecov Notifications / codecov/patch

tasks/tests/unit/test_test_results_processor_task.py#L568-L570

Added lines #L568 - L570 were not covered by tests
storage_path=second_url, report=second_report_row
)
dbsession.add(upload)
dbsession.flush()

Check warning on line 574 in tasks/tests/unit/test_test_results_processor_task.py

View check run for this annotation

Codecov Notifications / codecov/patch

tasks/tests/unit/test_test_results_processor_task.py#L573-L574

Added lines #L573 - L574 were not covered by tests

assert [r.fail_count for r in rollups] == [1, 0, 0, 1]
assert [r.pass_count for r in rollups] == [1, 1, 2, 0]
assert [r.skip_count for r in rollups] == [0, 0, 0, 0]
assert [r.flaky_fail_count for r in rollups] == [0, 0, 0, 1]
tests = dbsession.query(Test).all()
for test in tests:
flake = FlakeFactory.create(test=test)
dbsession.add(flake)
dbsession.flush()

Check warning on line 580 in tasks/tests/unit/test_test_results_processor_task.py

View check run for this annotation

Codecov Notifications / codecov/patch

tasks/tests/unit/test_test_results_processor_task.py#L576-L580

Added lines #L576 - L580 were not covered by tests

assert [r.commits_where_fail for r in rollups] == [
["cd76b0821854a780b60012aed85af0a8263004ad"],
[],
[],
["bd76b0821854a780b60012aed85af0a8263004ad"],
]
redis_queue = [{"url": second_url, "upload_pk": upload.id_}]

Check warning on line 582 in tasks/tests/unit/test_test_results_processor_task.py

View check run for this annotation

Codecov Notifications / codecov/patch

tasks/tests/unit/test_test_results_processor_task.py#L582

Added line #L582 was not covered by tests

assert [r.latest_run for r in rollups] == [
datetime(1970, 1, 1, 0, 0, tzinfo=timezone.utc),
datetime(1970, 1, 1, 0, 0, tzinfo=timezone.utc),
datetime(1970, 1, 2, 0, 0, tzinfo=timezone.utc),
datetime(1970, 1, 2, 0, 0, tzinfo=timezone.utc),
]
assert [r.avg_duration_seconds for r in rollups] == [0.001, 7.2, 0.002, 3.6]
assert [r.last_duration_seconds for r in rollups] == [0.001, 7.2, 0.002, 3.6]
traveller.stop()
result = TestResultsProcessorTask().run_impl(

Check warning on line 584 in tasks/tests/unit/test_test_results_processor_task.py

View check run for this annotation

Codecov Notifications / codecov/patch

tasks/tests/unit/test_test_results_processor_task.py#L584

Added line #L584 was not covered by tests
dbsession,
repoid=repoid,
commitid=second_commit.commitid,
commit_yaml={"codecov": {"max_report_age": False}},
arguments_list=redis_queue,
)
expected_result = [

Check warning on line 591 in tasks/tests/unit/test_test_results_processor_task.py

View check run for this annotation

Codecov Notifications / codecov/patch

tasks/tests/unit/test_test_results_processor_task.py#L591

Added line #L591 was not covered by tests
{
"successful": True,
}
]

rollups: list[DailyTestRollup] = dbsession.query(DailyTestRollup).all()

Check warning on line 597 in tasks/tests/unit/test_test_results_processor_task.py

View check run for this annotation

Codecov Notifications / codecov/patch

tasks/tests/unit/test_test_results_processor_task.py#L597

Added line #L597 was not covered by tests

assert result == expected_result

Check warning on line 599 in tasks/tests/unit/test_test_results_processor_task.py

View check run for this annotation

Codecov Notifications / codecov/patch

tasks/tests/unit/test_test_results_processor_task.py#L599

Added line #L599 was not covered by tests

assert [r.branch for r in rollups] == [

Check warning on line 601 in tasks/tests/unit/test_test_results_processor_task.py

View check run for this annotation

Codecov Notifications / codecov/patch

tasks/tests/unit/test_test_results_processor_task.py#L601

Added line #L601 was not covered by tests
"first_branch",
"first_branch",
"second_branch",
"second_branch",
]

assert [r.date for r in rollups] == [

Check warning on line 608 in tasks/tests/unit/test_test_results_processor_task.py

View check run for this annotation

Codecov Notifications / codecov/patch

tasks/tests/unit/test_test_results_processor_task.py#L608

Added line #L608 was not covered by tests
date.today() - timedelta(days=1),
date.today() - timedelta(days=1),
date.today(),
date.today(),
]

assert [r.fail_count for r in rollups] == [1, 0, 0, 1]
assert [r.pass_count for r in rollups] == [1, 1, 2, 0]
assert [r.skip_count for r in rollups] == [0, 0, 0, 0]
assert [r.flaky_fail_count for r in rollups] == [0, 0, 0, 1]

Check warning on line 618 in tasks/tests/unit/test_test_results_processor_task.py

View check run for this annotation

Codecov Notifications / codecov/patch

tasks/tests/unit/test_test_results_processor_task.py#L615-L618

Added lines #L615 - L618 were not covered by tests

assert [r.commits_where_fail for r in rollups] == [

Check warning on line 620 in tasks/tests/unit/test_test_results_processor_task.py

View check run for this annotation

Codecov Notifications / codecov/patch

tasks/tests/unit/test_test_results_processor_task.py#L620

Added line #L620 was not covered by tests
["cd76b0821854a780b60012aed85af0a8263004ad"],
[],
[],
["bd76b0821854a780b60012aed85af0a8263004ad"],
]

assert [r.latest_run for r in rollups] == [

Check warning on line 627 in tasks/tests/unit/test_test_results_processor_task.py

View check run for this annotation

Codecov Notifications / codecov/patch

tasks/tests/unit/test_test_results_processor_task.py#L627

Added line #L627 was not covered by tests
datetime(1970, 1, 1, 0, 0, tzinfo=timezone.utc),
datetime(1970, 1, 1, 0, 0, tzinfo=timezone.utc),
datetime(1970, 1, 2, 0, 0, tzinfo=timezone.utc),
datetime(1970, 1, 2, 0, 0, tzinfo=timezone.utc),
]
assert [r.avg_duration_seconds for r in rollups] == [0.001, 7.2, 0.002, 3.6]
assert [r.last_duration_seconds for r in rollups] == [

Check warning on line 634 in tasks/tests/unit/test_test_results_processor_task.py

View check run for this annotation

Codecov Notifications / codecov/patch

tasks/tests/unit/test_test_results_processor_task.py#L633-L634

Added lines #L633 - L634 were not covered by tests
0.001,
7.2,
0.002,
3.6,
]

@pytest.mark.integration
@pytest.mark.parametrize(
Expand Down

0 comments on commit 74abdd5

Please sign in to comment.