Merge pull request #72 from HopkinsIDD/config-v3

New simplified config format
HopkinsIDD · Oct 9, 2023 · 4de7873 · 4de7873
2 parents 102d5f1 + 1522c75
commit 4de7873
Show file tree

Hide file tree

Showing 70 changed files with 11,602 additions and 12,661 deletions.
diff --git a/batch/SLURM_inference_job.run b/batch/SLURM_inference_job.run
@@ -110,8 +110,8 @@ echo "***************** RUNNING INFERENCE_MAIN.R *****************"
 export LOG_FILE="$FS_RESULTS_PATH/log_${FLEPI_RUN_INDEX}_${FLEPI_SLOT_INDEX}.txt"
 echo "Rscript $FLEPI_PATH/flepimop/main_scripts/inference_slot.R --config $CONFIG_PATH   # path to the config file
                                                                  --run_id $FLEPI_RUN_INDEX  # Unique identifier for this run
-                                                                 --npi_scenarios $FLEPI_NPI_SCENARIOS  # name of the intervention to run, or 'all'
-                                                                 --outcome_scenarios $FLEPI_OUTCOME_SCENARIOS  # name of the outcome scenarios to run, or 'all'
+                                                                 --seir_modifiers_scenarios $FLEPI_NPI_SCENARIOS  # name of the intervention to run, or 'all'
+                                                                 --outcome_modifiers_scenarios $FLEPI_OUTCOME_SCENARIOS  # name of the outcome scenarios to run, or 'all'
                                                                  --jobs 1  # Number of jobs to run in parallel
                                                                  --iterations_per_slot $FLEPI_ITERATIONS_PER_SLOT # number of simulations to run for this slot
                                                                  --this_slot $FLEPI_SLOT_INDEX # id of this slot
@@ -125,7 +125,7 @@ echo "Rscript $FLEPI_PATH/flepimop/main_scripts/inference_slot.R --config $CONFI
                                                                  --is-resume $RESUME_RUN # Is this run a resume
                                                                  --is-interactive FALSE # Is this run an interactive run" #> $LOG_FILE 2>&1 &
 
-Rscript $FLEPI_PATH/flepimop/main_scripts/inference_slot.R -p $FLEPI_PATH --config $CONFIG_PATH --run_id $FLEPI_RUN_INDEX --npi_scenarios $FLEPI_NPI_SCENARIOS --outcome_scenarios $FLEPI_OUTCOME_SCENARIOS --jobs 1 --iterations_per_slot $FLEPI_ITERATIONS_PER_SLOT --this_slot $FLEPI_SLOT_INDEX --this_block 1 --stoch_traj_flag $FLEPI_STOCHASTIC_RUN --is-resume $RESUME_RUN --is-interactive FALSE #> $LOG_FILE 2>&1
+Rscript $FLEPI_PATH/flepimop/main_scripts/inference_slot.R -p $FLEPI_PATH --config $CONFIG_PATH --run_id $FLEPI_RUN_INDEX --seir_modifiers_scenarios $FLEPI_NPI_SCENARIOS --outcome_modifiers_scenarios $FLEPI_OUTCOME_SCENARIOS --jobs 1 --iterations_per_slot $FLEPI_ITERATIONS_PER_SLOT --this_slot $FLEPI_SLOT_INDEX --this_block 1 --stoch_traj_flag $FLEPI_STOCHASTIC_RUN --is-resume $RESUME_RUN --is-interactive FALSE #> $LOG_FILE 2>&1
 dvc_ret=$?
 if [[ $dvc_ret -ne 0 ]]; then
         echo "Error code returned from inference_slot.R: $dvc_ret"

diff --git a/batch/SLURM_inference_runner.sh b/batch/SLURM_inference_runner.sh
@@ -87,8 +87,8 @@ echo "***************** RUNNING inference_slot.R *****************"
 export LOG_FILE="$FS_RESULTS_PATH/log_${FLEPI_RUN_INDEX}_${FLEPI_SLOT_INDEX}.txt"
 echo "Rscript $FLEPI_PATH/flepimop/main_scripts/inference_slot.R --config $CONFIG_PATH   # path to the config file
                                                --run_id $FLEPI_RUN_INDEX  # Unique identifier for this run
-                                               --npi_scenarios $FLEPI_NPI_SCENARIOS  # name of the intervention to run, or 'all'
-                                               --outcome_scenarios $FLEPI_OUTCOME_SCENARIOS  # name of the outcome scenarios to run, or 'all'
+                                               --seir_modifiers_scenarios $FLEPI_NPI_SCENARIOS  # name of the intervention to run, or 'all'
+                                               --outcome_modifiers_scenarios $FLEPI_OUTCOME_SCENARIOS  # name of the outcome scenarios to run, or 'all'
                                                --jobs 1  # Number of jobs to run in parallel
                                                --iterations_per_slot $FLEPI_ITERATIONS_PER_SLOT # number of iterations to run for this slot
                                                --this_slot $FLEPI_SLOT_INDEX # id of this slot
@@ -102,7 +102,7 @@ echo "Rscript $FLEPI_PATH/flepimop/main_scripts/inference_slot.R --config $CONFI
                                                --is-resume $RESUME_RUN # Is this run a resume
                                                --is-interactive FALSE # Is this run an interactive run" > $LOG_FILE 2>&1 &
 
-Rscript $FLEPI_PATH/flepimop/main_scripts/inference_slot.R -p $FLEPI_PATH --this_slot $FLEPI_SLOT_INDEX --config $CONFIG_PATH --run_id $FLEPI_RUN_INDEX --npi_scenarios $FLEPI_NPI_SCENARIOS --outcome_scenarios $FLEPI_OUTCOME_SCENARIOS --jobs 1 --iterations_per_slot $FLEPI_ITERATIONS_PER_SLOT --this_block 1 --stoch_traj_flag $FLEPI_STOCHASTIC_RUN --is-resume $RESUME_RUN --is-interactive FALSE > $LOG_FILE 2>&1
+Rscript $FLEPI_PATH/flepimop/main_scripts/inference_slot.R -p $FLEPI_PATH --this_slot $FLEPI_SLOT_INDEX --config $CONFIG_PATH --run_id $FLEPI_RUN_INDEX --seir_modifiers_scenarios $FLEPI_NPI_SCENARIOS --outcome_modifiers_scenarios $FLEPI_OUTCOME_SCENARIOS --jobs 1 --iterations_per_slot $FLEPI_ITERATIONS_PER_SLOT --this_block 1 --stoch_traj_flag $FLEPI_STOCHASTIC_RUN --is-resume $RESUME_RUN --is-interactive FALSE > $LOG_FILE 2>&1
 dvc_ret=$?
 if [ $dvc_ret -ne 0 ]; then
         echo "Error code returned from inference_main.R: $dvc_ret"

diff --git a/batch/inference_job_launcher.py b/batch/inference_job_launcher.py
@@ -300,7 +300,6 @@ def launch_batch(
     continuation_location,
     continuation_run_id,
 ):
-
     config = None
     with open(config_file) as f:
         config = yaml.full_load(f)
@@ -393,10 +392,10 @@ def launch_batch(
         continuation_run_id,
     )
 
-    npi_scenarios = config["interventions"]["scenarios"]
-    outcome_scenarios = config["outcomes"]["scenarios"]
+    seir_modifiers_scenarios = config["seir_modifiers"]["scenarios"]
+    outcome_modifiers_scenarios = config["outcomes"]["scenarios"]
 
-    handler.launch(job_name, config_file, npi_scenarios, outcome_scenarios)
+    handler.launch(job_name, config_file, seir_modifiers_scenarios, outcome_modifiers_scenarios)
 
     # Set job_name as environmental variable so it can be pulled for pushing to git
     os.environ["job_name"] = job_name
@@ -648,7 +647,7 @@ def save_file(self, source, destination, remove_source=False, prefix=""):
         if remove_source:
             os.remove(source)
 
-    def launch(self, job_name, config_file, npi_scenarios, outcome_scenarios):
+    def launch(self, job_name, config_file, seir_modifiers_scenarios, outcome_modifiers_scenarios):
         s3_results_path = f"s3://{self.s3_bucket}/{job_name}"
 
         if self.batch_system == "slurm":
@@ -699,7 +698,7 @@ def launch(self, job_name, config_file, npi_scenarios, outcome_scenarios):
         with open(config_file) as f:
             config = yaml.full_load(f)
 
-        for ctr, (s, d) in enumerate(itertools.product(npi_scenarios, outcome_scenarios)):
+        for ctr, (s, d) in enumerate(itertools.product(seir_modifiers_scenarios, outcome_modifiers_scenarios)):
             cur_job_name = f"{job_name}_{s}_{d}"
             # Create first job
             cur_env_vars = base_env_vars.copy()

diff --git a/batch/scenario_job.py b/batch/scenario_job.py
@@ -121,7 +121,6 @@ def launch_batch(
     vcpu,
     memory,
 ):
-
     config = None
     with open(config_file) as f:
         config = yaml.full_load(f)
@@ -134,14 +133,14 @@ def launch_batch(
     config["nslots"] = slots_per_job
 
     if parallelize_scenarios:
-        npi_scenarios = config["interventions"]["scenarios"]
-        for s in npi_scenarios:
-            npi_scenario_job_name = f"{job_name}_{s}"
+        seir_modifiers_scenarios = config["seir_modifiers"]["scenarios"]
+        for s in seir_modifiers_scenarios:
+            seir_modifiers_scenario_job_name = f"{job_name}_{s}"
             config["interventions"]["scenarios"] = [s]
             with open(config_file, "w") as f:
                 yaml.dump(config, f, sort_keys=False)
             launch_job_inner(
-                npi_scenario_job_name,
+                seir_modifiers_scenario_job_name,
                 config_file,
                 num_jobs,
                 slots_per_job,
@@ -153,7 +152,7 @@ def launch_batch(
                 vcpu,
                 memory,
             )
-        config["interventions"]["scenarios"] = npi_scenarios
+        config["interventions"]["scenarios"] = seir_modifiers_scenarios
         with open(config_file, "w") as f:
             yaml.dump(config, f, sort_keys=False)
     else:
@@ -191,7 +190,6 @@ def launch_job_inner(
     vcpu,
     memory,
 ):
-
     # Prepare to tar up the current directory, excluding any dvc outputs, so it
     # can be shipped to S3
     dvc_outputs = get_dvc_outputs()

diff --git a/flepimop/R_packages/config.writer/NAMESPACE b/flepimop/R_packages/config.writer/NAMESPACE
@@ -45,8 +45,8 @@ export(set_vacc_rates_params)
 export(set_vacc_rates_params_dose3)
 export(set_variant_params)
 export(set_ve_shift_params)
-export(yaml_mtr_template)
-export(yaml_reduce_template)
+export(yaml_mtr_method)
+export(yaml_reduce_method)
 export(yaml_stack1)
 export(yaml_stack2)
 importFrom(magrittr,"%>%")