Skip to content

Commit

Permalink
correctly spawn parallel experiment tasks
Browse files Browse the repository at this point in the history
  • Loading branch information
Swatinem committed Sep 10, 2024
1 parent f1ab443 commit e971d76
Showing 1 changed file with 7 additions and 1 deletion.
8 changes: 7 additions & 1 deletion tasks/upload.py
Original file line number Diff line number Diff line change
Expand Up @@ -604,6 +604,7 @@ def _schedule_coverage_processing_task(
commit_report,
upload_context,
checkpoints,
parallel_processing.run_fully_parallel,
)

if parallel_processing.run_fully_parallel:
Expand Down Expand Up @@ -654,6 +655,7 @@ def create_parallel_tasks(
commit_report: CommitReport,
upload_context: UploadContext,
checkpoints: CheckpointLogger,
run_fully_parallel: bool,
):
report_service = ReportService(commit_yaml)
sessions = report_service.build_sessions(commit=commit)
Expand Down Expand Up @@ -706,7 +708,6 @@ def create_parallel_tasks(

parallel_processing_tasks = [
upload_processor_task.s(
{},
repoid=commit.repoid,
commitid=commit.commitid,
commit_yaml=commit_yaml,
Expand All @@ -720,6 +721,11 @@ def create_parallel_tasks(
argument_list, parallel_session_ids
)
]
if run_fully_parallel:
for task in parallel_processing_tasks:
# this is the `previous_results`, which celery provides when running
# in a chain as part of the experiment, otherwise we have to provide this.
task.args = ({},)

finish_parallel_sig = upload_finisher_task.signature(
kwargs={
Expand Down

0 comments on commit e971d76

Please sign in to comment.