Improve multiprocessor; autoformat with black

app.py

@@ -142,10 +142,7 @@ def process_excel_files(
         if preview_experiment_files:
             print("HERE, preview_experiment_files")
             compare_files(
-                experiment_files,
-                preview_experiment_files,
-                project_folder,
-                times=times,
+                experiment_files, preview_experiment_files, project_folder, times=times,
             )
         else:
             print("global_plots")
@@ -169,7 +166,12 @@ def process_excel_files(
     processed_files = []
     for idx, c in enumerate([control_file_1, control_file_2]):
         C = ControlFile(
-            flush, wait, close, c, file_type=f"control_{idx + 1}", ignore_loops=ignore_loops,
+            flush,
+            wait,
+            close,
+            c,
+            file_type=f"control_{idx + 1}",
+            ignore_loops=ignore_loops,
         )
         C_Total = ResumeControl(C)
         C_Total.get_bank()
@@ -180,27 +182,30 @@ def process_excel_files(
     now = time.perf_counter()
     for i, data_file in enumerate(uploaded_excel_files):
         experiment = ExperimentCycle(
-            flush, wait, close, data_file, ignore_loops=ignore_loops, file_type="Experiment",
+            flush,
+            wait,
+            close,
+            data_file,
+            ignore_loops=ignore_loops,
+            file_type="Experiment",
         )
         resume = ResumeDataFrame(experiment)
         resume.generate_resume(control)
         resume.save()
         processed_files.append(experiment)
 
+    p = Pool()
+    if plot:
+        p.map(save_loop_graph, processed_files)
     if config["experiment_file_config"]["SAVE_LOOP_DF"]:
-        # p = Process(target=save_loop_file, name="SaveLoops", args=(processed_files,))
-        p = Pool()
         p.map(save_loop_file, processed_files)
-        if plot:
-            p.map(save_loop_graph, processed_files)
-
     if config["experiment_file_config"]["SAVE_CONVERTED"]:
         for f in processed_files:
             f.original_file.save(name=f"[Original]{f.original_file.fname}")
 
     TearDown(Path(experiment.original_file.folder_dst)).organize()
     cache.set("generating_files", False)
-    print("Tasca conclosa")
+    print("✨ Tasca conclosa ✨")
     print(f"Processament de temps total {round(time.perf_counter() - now, 3)} segons")
 
 
@@ -247,7 +252,9 @@ def excel_files():
         flush = int(request.form.get("flush"))
         wait = int(request.form.get("wait"))
         close = int(request.form.get("close"))
-        plot = True if request.form.get("plot") else False  # if generate or no loop plots
+        plot = (
+            True if request.form.get("plot") else False
+        )  # if generate or no loop plots
         # Show preview plot if user wants
         control_file_1.filename = "C1.txt"
         control_file_2.filename = "C2.txt"
@@ -272,7 +279,9 @@ def excel_files():
         try:
             os.mkdir(project_folder)
         except FileExistsError:
-            project_folder = os.path.join(app.config["UPLOAD_FOLDER"], f"{folder_name}_1")
+            project_folder = os.path.join(
+                app.config["UPLOAD_FOLDER"], f"{folder_name}_1"
+            )
             os.mkdir(project_folder)
         # Save all files into project folder
         files_list = [data_file, control_file_1, control_file_2]
@@ -291,7 +300,8 @@ def excel_files():
         # save the full path of the saved file
         uploaded_excel_files.append(os.path.join(project_folder, data_file.filename))
         t = Thread(
-            target=process_excel_files, args=(flush, wait, close, uploaded_excel_files, plot),
+            target=process_excel_files,
+            args=(flush, wait, close, uploaded_excel_files, plot),
         )
         t.start()
 
@@ -361,7 +371,9 @@ def downloads():
 
 @app.route("/downloads/all")
 def download_all():
-    all_zipped = shutil.make_archive(app.config["ZIP_FOLDER"], "zip", app.config["ZIP_FOLDER"])
+    all_zipped = shutil.make_archive(
+        app.config["ZIP_FOLDER"], "zip", app.config["ZIP_FOLDER"]
+    )
     return send_from_directory(Path(all_zipped).parent, Path(all_zipped).name)
 
 
@@ -407,7 +419,7 @@ def help():
 @app.route("/status", methods=["GET"])
 def get_status():
     """Return information about the different components of the system."""
-    return jsonify({"generating_files": cache.get("generating_files"),})
+    return jsonify({"generating_files": cache.get("generating_files")})
 
 
 @app.route("/ignore_loops/<data>", methods=["POST"])
@@ -450,13 +462,15 @@ def ignored_loops():
     print("*" * 70)
     print("\n")
     print("Carregant l'aplicació ...")
-    # webbrowser.open(f"http://localhost:{port}")
+    webbrowser.open(f"http://localhost:{port}")
     print("\n")
-    print("Si l'aplicació no s'obre automàticament, introduïu la següent URL al navegador")
+    print(
+        "Si l'aplicació no s'obre automàticament, introduïu la següent URL al navegador"
+    )
     print(f"http://localhost:{port}")
     print("\n")
     print("*" * 70)
     print("Avís: tancant aquesta finestra es tancarà l’aplicació")
     print("*" * 70)
-    socketio.run(app, debug=False, host="0.0.0.0", port=port)
-    # socketio.run(app, debug=True, host="0.0.0.0", port=port)
+    # socketio.run(app, debug=False, host="0.0.0.0", port=port)
+    socketio.run(app, debug=True, host="0.0.0.0", port=port)

config.json

@@ -1 +1 @@
-{"experiment_file_config": {"PLOT_TITLE": "Evoluci\u00f3 de l'oxigen disde el aire", "X_COL": "Temps (min)", "Y_COL": "mg 02/L", "DT_COL": "Date &Time [DD-MM-YYYY HH:MM:SS]", "TSCODE": "Time stamp code", "O2_COL": "SDWA0003000061      , CH 1 O2 [mg/L]", "SAVE_LOOP_DF": true, "SAVE_CONVERTED": true}, "file_cycle_config": {"flush": 3, "wait": 10, "close": 40, "aqua_volume": 0.2}, "pump_control_config": {}}
+{"experiment_file_config": {"PLOT_TITLE": "Evoluci\u00f3 de l'oxigen disde el aire", "X_COL": "Temps (min)", "Y_COL": "mg 02/L", "DT_COL": "Date &Time [DD-MM-YYYY HH:MM:SS]", "TSCODE": "Time stamp code", "O2_COL": "SDWA0003000061      , CH 1 O2 [mg/L]", "SAVE_LOOP_DF": true, "SAVE_CONVERTED": true}, "file_cycle_config": {"flush": 3, "wait": 10, "close": 40, "aqua_volume": 0.2}, "pump_control_config": {}}

core/converter.py

@@ -268,7 +268,7 @@ def _close_df(self, start: datetime, end: datetime) -> pd.DataFrame:
         df_close.loc[:, self.y] = df_close[self.O2_COL].map(string_to_float)
         return df_close
 
-    #@progress_bar
+    # @progress_bar
     def create_plot(self, format_="html"):
         """Proxy for Plot object."""
         print("Generació de gràfics", end="\n")
@@ -283,7 +283,7 @@ def create_plot(self, format_="html"):
                 dst=os.path.dirname(self.original_file.file_output),
                 fname=f"{self.original_file.fname}_loop{k}",
             ).create()
-            #yield round(step * k)
+            # yield round(step * k)
 
     def save(self, df_loop, name):
         """Save data frame into a excel file."""

core/resume.py

@@ -80,9 +80,7 @@ def __init__(self, experiment):
         self.experiment = experiment
         self.dt_col_name = experiment.dt_col_name
         self.df_lists = []
-        self.phase_time = (
-            f"F{experiment.flush*60}/W{experiment.wait*60}/C{experiment.close*60}"  # noqa
-        )
+        self.phase_time = f"F{experiment.flush*60}/W{experiment.wait*60}/C{experiment.close*60}"  # noqa
 
     @property
     def experiment_files(self) -> list:
@@ -112,7 +110,10 @@ def generate_resume(self, control):
         for i, df_close in enumerate(self.experiment.df_loop_generator):
             k = i + 1
             try:
-                if str(k) in self.experiment.ignore_loops[self.experiment.original_file.fname]:
+                if (
+                    str(k)
+                    in self.experiment.ignore_loops[self.experiment.original_file.fname]
+                ):
                     continue
             except KeyError:
                 pass
@@ -135,7 +136,9 @@ def generate_resume(self, control):
                 "max O2 [mgO2/L]": O2.max,
                 "min O2 [mgO2/L]": O2.min,
                 "avg O2 [mgO2/L]": O2.avg,
-                "avg temp [°C]": temp_mean(df_close["SDWA0003000061      , CH 1 temp [°C]"]),
+                "avg temp [°C]": temp_mean(
+                    df_close["SDWA0003000061      , CH 1 temp [°C]"]
+                ),
                 "O2 after blank": O2_HR - control,
             }
 
@@ -169,7 +172,8 @@ def get_bank(self):
         self.save()
         try:
             self.values.append(
-                self.resume_df["MO2 [mgO2/hr]"].sum() / len(self.resume_df["MO2 [mgO2/hr]"])
+                self.resume_df["MO2 [mgO2/hr]"].sum()
+                / len(self.resume_df["MO2 [mgO2/hr]"])
             )
         except ZeroDivisionError:
             print("Control table empty")

core/utils.py

@@ -24,6 +24,7 @@ def global_plots(
 ):
     """Proxy function to deal with global graphs."""
     from core.converter import ExperimentCycle
+
     if not keep:
         for f in files:
             file_path = str(Path(preview_folder) / f.filename)
@@ -36,7 +37,9 @@ def global_plots(
             # os.remove(file_path)
     else:
         for f in files:
-            experiment = ExperimentCycle(flush, wait, close, f, file_type="Global grafic")
+            experiment = ExperimentCycle(
+                flush, wait, close, f, file_type="Global grafic"
+            )
             experiment.experiment_plot()
 
 

tests/UI_test.py

@@ -1,9 +1,11 @@
 from pathlib import Path
 import pytest
+import requests
 from selenium import webdriver
-from selenium.webdriver.common.by import By
+from selenium.common.exceptions import ElementNotInteractableException
+from core.utils import config_from_file, save_config_to_file
 
-test_data = Path(".").resolve() /"tests/Data"
+test_data = Path(".").resolve() / "tests/Data"
 data_file = test_data / "angula.txt"
 control_file_1 = test_data / "control1.txt"
 control_file_2 = test_data / "control2.txt"
@@ -21,8 +23,6 @@ def driver():
 # @pytest.mark.skip
 def test_home_page(driver):
     driver.get("http://localhost:5000/")
-    # starting = driver.find_element_by_id("get_starting")
-    # assert starting.text == "Lets map!"
 
 
 def process_data_with_graphics(preview=False):
@@ -40,18 +40,28 @@ def process_data_with_graphics(preview=False):
     else:
         preview_plots(driver)
 
+
 def preview_plots(driver):
     vista_previa = driver.find_element_by_id("generatePlotComplet")
     vista_previa.click()
     submit = driver.find_element_by_id("submit_files")
+    submit.click()
     # Preview plots
-    driver.implicitly_wait(5)
+    driver.implicitly_wait(10)
     driver.find_element_by_id("C1").send_keys("1,2")
     driver.find_element_by_id("C1ID").click()
-    driver.find_element_by_id("myBtn").click()
+    try:
+        driver.find_element_by_id("myBtn").click()
+    except ElementNotInteractableException:
+        pass
     driver.find_element_by_xpath("/html/body/nav/a[2]").click()
-    driver.implicitly_wait(0.1)
+    driver.implicitly_wait(0.5)
     driver.find_element_by_id("generatePlots").click()
-    submit.click()
+    driver.find_element_by_id("submit_files").click()
+
+
+def process_preview_and_plot():
+    return process_data_with_graphics(True)
+
 
-process_data_with_graphics()
+process_preview_and_plot()

tests/test_routes.py

@@ -116,12 +116,15 @@ def test_file_upload_graphic():
     pattern = r"\w+\.zip"
     fname = re.search(pattern, response.text)
     project_folder = fname.group().split(".")[0]
-    preview_txt_files = [str(f) for f in (ROOT_DIR / "static/uploads/preview").glob("*.txt")]
+    preview_txt_files = [
+        str(f) for f in (ROOT_DIR / "static/uploads/preview").glob("*.txt")
+    ]
     sorted(preview_txt_files)
 
     # Upload and generate all graphics
     uploaded_files = [
-        str(f) for f in (ROOT_DIR / f"static/uploads/angula_24_02_2020_18_24_06").glob("*.txt")
+        str(f)
+        for f in (ROOT_DIR / f"static/uploads/angula_24_02_2020_18_24_06").glob("*.txt")
     ]
     sorted(uploaded_files)
     for preview, uploaded in zip(uploaded_files, preview_txt_files):
@@ -133,6 +136,7 @@ def test_file_upload_graphic():
             # {fname_gplot: True}
             pass
 
+
 @pytest.mark.skip
 def test_main_page():
     response = requests.get(URL)