Analysis refactor

scripts/dir_exp_analysis.py

@@ -67,4 +67,3 @@ def main(output_path: str):
 
 if __name__ == "__main__":
     CLI(main)
-    CLI(main)

scripts/experiment_analysis.py

@@ -9,7 +9,7 @@
     plot_vectors,
     single_model_analysis,
 )
-from arc_spice.eval.analysis_utils import brier_score
+from arc_spice.analysis.utils import brier_score
 
 
 def main(

scripts/propagation_analysis.py

@@ -6,17 +6,7 @@
 
 from jsonargparse import CLI
 
-from arc_spice.eval.analysis_utils import (
-    collect_pipeline_dict,
-    exp_analysis,
-    test_train_split_res,
-)
-from arc_spice.eval.prop_models import (
-    eval_lin_models,
-    eval_mult_prop,
-    fit_uncertainty_model,
-)
-from arc_spice.utils import open_json_path
+from arc_spice.analysis.analysis_functions import propagation_analysis
 
 
 def main(experiment_path: str):
@@ -26,35 +16,7 @@ def main(experiment_path: str):
     Args:
         experiment_path: path experiment directory
     """
-    model_keys = ["ocr", "translator", "classifier"]
-
-    # collect and collate results
-    pipeline_results = open_json_path(f"{experiment_path}/full_pipeline.json")
-    pipe_results = collect_pipeline_dict(pipeline_results)
-
-    # no model results, rename keys
-    no_mod_res = exp_analysis(pipe_results, model_keys)
-    no_mod_res["recognition"] = no_mod_res.pop("ocr")
-    no_mod_res["translation"] = no_mod_res.pop("translator")
-    no_mod_res["classification"] = no_mod_res.pop("classifier")
-
-    # multplication model resuls
-    multi_mod_res = eval_mult_prop(pipe_results)
-
-    # fitted model results
-    train_res, test_res = test_train_split_res(pipe_results)
-    fitted_uq_models = fit_uncertainty_model(train_res)
-    fit_mod_res = eval_lin_models(fitted_uq_models, test_res)
-
-    # collate results
-    out_res = {}
-    for key, itm in fit_mod_res.items():
-        out_res[key] = {
-            "no_model": no_mod_res[key],
-            "mult_model": multi_mod_res[key],
-            "fitted_model": itm,
-        }
-
+    out_res = propagation_analysis(experiment_path)
     # save results
     with open(f"{experiment_path}/prop_model_analysis.json", "w") as save_file:
         json.dump(out_res, save_file, indent=2)

src/arc_spice/analysis/analysis_functions.py

@@ -1,11 +1,60 @@
 import matplotlib.pyplot as plt
 import numpy as np
 
-from arc_spice.analysis.utils import fitted_lin_model, multiplication_prop
-from arc_spice.eval.analysis_utils import exp_analysis, exp_vectors
+from arc_spice.analysis.prop_models import (
+    eval_lin_models,
+    eval_mult_prop,
+    fit_uncertainty_model,
+    fitted_lin_model,
+    multiplication_prop,
+)
+from arc_spice.analysis.utils import (
+    collect_pipeline_dict,
+    exp_analysis,
+    exp_vectors,
+    test_train_split_res,
+)
 from arc_spice.utils import open_json_path
 
 
+def propagation_analysis(experiment_path: str):
+    """Run analysis of a given pipeline experiment using the different propagation
+    models
+
+    Args:
+        experiment_path: path experiment directory
+    """
+    model_keys = ["ocr", "translator", "classifier"]
+
+    # collect and collate results
+    pipeline_results = open_json_path(f"{experiment_path}/full_pipeline.json")
+    pipe_results = collect_pipeline_dict(pipeline_results)
+
+    # no model results, rename keys
+    no_mod_res = exp_analysis(pipe_results, model_keys)
+    no_mod_res["recognition"] = no_mod_res.pop("ocr")
+    no_mod_res["translation"] = no_mod_res.pop("translator")
+    no_mod_res["classification"] = no_mod_res.pop("classifier")
+
+    # multplication model resuls
+    multi_mod_res = eval_mult_prop(pipe_results)
+
+    # fitted model results
+    train_res, test_res = test_train_split_res(pipe_results)
+    fitted_uq_models = fit_uncertainty_model(train_res)
+    fit_mod_res = eval_lin_models(fitted_uq_models, test_res)
+
+    # collate results
+    out_res = {}
+    for key, itm in fit_mod_res.items():
+        out_res[key] = {
+            "no_model": no_mod_res[key],
+            "mult_model": multi_mod_res[key],
+            "fitted_model": itm,
+        }
+    return out_res
+
+
 def single_model_analysis(
     experiment_path: str,
 ):
@@ -108,6 +157,7 @@ def plot_vectors(
     plt.legend()
     plt.xlim(0, 1)
     plt.savefig(f"{save_directory}/figures/recognition_confidence_histogram.pdf")
+    plt.close()
 
     # Translation
     fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(8, 12))
@@ -208,6 +258,7 @@ def plot_vectors(
     ax1.set_xlim(0, 1)
     ax2.set_xlim(0, 1)
     plt.savefig(f"{save_directory}/figures/translation_confidence_histogram.pdf")
+    plt.close()
 
     fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(8, 12))
 
@@ -290,7 +341,4 @@ def plot_vectors(
     ax1.set_xlim(0, 1)
     ax2.set_xlim(0, 1)
     plt.savefig(f"{save_directory}/figures/classification_confidence_histogram.pdf")
-
-    ax1.set_xlim(0, 1)
-    ax2.set_xlim(0, 1)
-    plt.savefig(f"{save_directory}/figures/classification_confidence_histogram.pdf")
+    plt.close()