Update from dev branch

.gitignore

@@ -1,3 +1,5 @@
+build
+dist
 venv
 *.pyc
 *.xlsx

TODO.md

@@ -7,5 +7,10 @@ FrontEnd
  - [x] Add control form inputs
  - [x] Add cache keys/values using set_many
  - [x] Allows save DF loops and converted file
- - [] Add possibility to save plot as png
+ - [ ] change original file  name when saved
+ - [x] add info led on title
+ - [ ] Delete volume from pump control settings
+ - [ ] Add setting to change default saved DF file name
+ - [ ] Add setting to change default saved plot file name
+ - [ ] Add possibility to save plot as png
  - [ ] Improve plot

app.py

@@ -91,7 +91,8 @@
     maxBytes=app.config["LOGS_MB_SIZE"],
     backupCount=app.config["LOGS_BACKUP"],
 )
-handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(message)s"))
+# handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(message)s"))
+handler.setFormatter(logging.Formatter("%(message)s"))
 handler.setLevel(logging.WARNING)
 
 app.logger.addHandler(handler)
@@ -152,7 +153,8 @@ def switch_off():
 def pump_cycle(cycle, period):
     """Define how long pump is ON in order to full the fish tank."""
     # Turn on the pump
-    started = datetime.now()
+    started = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    cache.set("total_loops", cache.get("total_loops") + 1)
     switch_on()
     cache.set("cycle_ends_in", to_js_time(cycle, "auto"))
     socketio.emit(
@@ -162,6 +164,8 @@ def pump_cycle(cycle, period):
             "running": cache.get("running"),
             "run_auto": cache.get("running"),
             "cycle_ends_in": cache.get("cycle_ends_in"),
+            "total_loops": cache.get("total_loops"),
+            "auto_run_since": cache.get("auto_run_since"),
         },
         namespace="/resPi",
     )
@@ -181,16 +185,17 @@ def pump_cycle(cycle, period):
                 },
                 namespace="/resPi",
             )
-            ended = datetime.now()
+            ended = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
             # print(f"Current automatic program: Started {str(started)} | Ended: {str(ended)}")
             # Write information to logging file
-            logger.warning(
-                f"Current automatic program: Started {str(started)} | Ended: {str(ended)}"
+            print(
+                f"""Current program [{cache.get("total_loops")}]: Started {started} | Ended: {ended}"""
             )
-        else:  # Ignore previous. Pump is already off
             logger.warning(
-                f"Automatic program: Started {str(started)} was closed forced by user"
+                f"""Current program [{cache.get("total_loops")}]: Started {started} | Ended: {ended}"""
             )
+        else:  # Ignore previous. Pump is already off
+            logger.warning(f"Automatic program: Started {started} was closed forced by user")
 
 
 ####################
@@ -199,12 +204,14 @@ def pump_cycle(cycle, period):
 # USER DEFINED PROGRAM
 def start_program(app=None):
     """Start a new background thread to run the user program."""
-    logger.warning("""Starting a new background thread to run the user program.""")
     # program()
     """User defined task.
 
     Creates a periodic task using user form input.
     """
+    # Save starting time programming
+    cache.set("auto_run_since", (datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
+
     user_program = cache.get("user_program")
     # Turn the pump on every x seconds
     period = (user_program.get("close") + user_program.get("wait")) * UNIT
@@ -220,7 +227,6 @@ def start_program(app=None):
 def process_excel_files(flush, wait, close, uploaded_excel_files, plot):
     """Start a new thread to process excel file uploaded by the user."""
     # Loop throw all uploaded files and clean the data set
-    print(f"{uploaded_excel_files=}")
     save_converted = False  # if to save .txt file converted into .xlsx file
 
     # CALCULATE BLANKS
@@ -289,7 +295,6 @@ def respi():
 
     This route contains all user interface possibilities with the hardware.
     """
-
     if request.method == "POST":
         # Get information from user form data and run automatic program
         if request.form.get("action", False) == "start":
@@ -303,13 +308,13 @@ def respi():
                 close = int(request.form["close"])
                 # set program configuration on memory layer
                 cache.set("user_program", dict(close=close, flush=flush, wait=wait))
+                cache.set("total_loops", 0)
                 session["user_program"] = [flush, wait, close]
                 # Create a register of the started thread
                 global _active_threads
                 t = Thread(target=start_program)
                 t_name = t.getName()
                 _active_threads[t_name] = t  # noqa
-                logger.warning(f"STARTED NEW {_active_threads}")
                 exit_thread.clear()  # set all thread flags to false
                 t.start()  # start a fresh new thread with the current program
         elif request.form.get("action", False) == "stop":
@@ -324,7 +329,6 @@ def respi():
                 )
             )
             exit_thread.set()
-            logger.warning(f"AFTER SET {_active_threads}")
         ###########################
         # MANUAL MODE
         ###########################
@@ -349,8 +353,6 @@ def respi():
         wait = cache.get("user_program")["wait"]
         close = cache.get("user_program")["close"]
 
-    logger.warning(f"2 AFTER SET {_active_threads}")
-
     return render_template("app.html", flush=flush, wait=wait, close=close)
 
 
@@ -376,7 +378,7 @@ def excel_files():
         uploaded_excel_files = []
         # for file_ in files:
         # Generate the folder name
-        time_stamp = datetime.now().strftime(f"%Y_%m_%d_%H_%M_%S")
+        time_stamp = datetime.now().strftime(f"%d_%m_%Y_%H_%M_%S")
         filename, ext = data_file.filename.split(".")
         if not check_extensions(ext):
             flash(
@@ -434,7 +436,6 @@ def excel_files():
 @app.route("/downloads", methods=["GET"])
 def downloads():
     """Route to see all zip files available to download."""
-    print(f"{cache.get_dict()=}")
     zip_folder = glob(f"{app.config['ZIP_FOLDER']}/*.zip")
     # get only the file name and the size of it excluding the path.
     # Create a list of tuples sorted by file name
@@ -472,7 +473,6 @@ def get_file(file_):
 def remove_file(file_):
     """Delete a zip file based on file name."""
     location = os.path.join(app.config["ZIP_FOLDER"], file_)
-    print(location)
     delete_zip_file(location)
     return redirect(url_for("downloads"))
 
@@ -569,7 +569,6 @@ def restart():
 @app.route("/status", methods=["GET"])
 def get_status():
     """Return information about the different components of the system."""
-    print("STATUS ", cache.get("started_at"))
     return jsonify(
         {
             "running": cache.get("running"),
@@ -579,9 +578,18 @@ def get_status():
             "cycle_ends_in": cache.get("cycle_ends_in"),
             "next_cycle_at": cache.get("next_cycle_at"),
             "generating_files": cache.get("generating_files"),
+            "total_loops": cache.get("total_loops"),
+            "auto_run_since": cache.get("auto_run_since"),
         }
     )
 
 
+@app.route("/user_time/<local_time>", methods=["GET", "POST"])
+def update_time(local_time):
+    """Get user local time to update server time."""
+    print(local_time)
+    return redirect(url_for("login"))
+
+
 if __name__ == "__main__":
     socketio.run(app, debug=True, host="0.0.0.0")

before-prod.sh

@@ -12,7 +12,7 @@ mkdir logs
 cd ../
 
 echo ------- sending to Pi -------
-scp -r resPI/ [email protected].0.11:/home/pi
+scp -r resPI/ [email protected].4.1:/home/pi
 
 echo ------- moving venv -------
 mv venv/ resPI/

config.json

@@ -1 +1 @@
-{"experiment_file_config": {"DT_COL": "Date &Time [DD-MM-YYYY HH:MM:SS]", "TSCODE": "Time stamp code", "O2_COL": "SDWA0003000061      , CH 1 O2 [mg/L]", "PLOT_TITLE": "O2 Evo", "X_COL": "data", "Y_COL": "o2", "SAVE_LOOP_DF": true, "SAVE_CONVERTED": true}, "file_cycle_config": {"flush": 3, "wait": 2, "close": 20, "aqua_volume": 21.0}, "pump_control_config": {"flush": 3, "wait": 2, "close": 20, "aqua_volume": "40.434"}}
+{"experiment_file_config": {"DT_COL": "Date &Time [DD-MM-YYYY HH:MM:SS]", "TSCODE": "Time stamp code", "O2_COL": "SDWA0003000061      , CH 1 O2 [mg/L]", "PLOT_TITLE": "Evoluci\u00f3 de l\u2019oxigen", "X_COL": "Time", "Y_COL": "O2", "SAVE_LOOP_DF": true, "SAVE_CONVERTED": true}, "file_cycle_config": {"flush": 3, "wait": 2, "close": 20, "aqua_volume": 0.2}, "pump_control_config": {"flush": 3, "wait": 2, "close": 20}}

scripts/__init__.py

@@ -0,0 +1,2 @@
+from .converter import ControlFile, ExperimentCycle, FileFormater, Plot  # noqa
+from .stats import ResumeDataFrame  # noqa

scripts/converter.py

@@ -109,9 +109,7 @@ class ExperimentCycle:
     dt_col_name: Name of that contains the experiment time spans
     """
 
-    def __init__(
-        self, flush: int, wait: int, close: int, original_file: str
-    ):  # noqa
+    def __init__(self, flush: int, wait: int, close: int, original_file: str):  # noqa
         self.flush = flush
         self.wait = wait
         self.close = close
@@ -213,33 +211,49 @@ def _close_df(self, start: datetime, end: datetime) -> pd.DataFrame:
         df_close[self.y] = df_close[self.O2_COL].map(string_to_float)
         return df_close
 
-    def create_plot(self):
+    def create_plot(self, format_="html"):
         for i, df_close in enumerate(self.df_close_list):
-            x = df_close[self.x]
-            y = df_close[self.y]
-            fig = go.Figure()
-            fig.add_trace(
-                go.Scatter(
-                    x=x,
-                    y=y,
-                    name=self.plot_title,
-                    line=dict(color="red", width=1),
-                    showlegend=True,
-                )
-            )
-            fig1 = px.scatter(df_close, x=x, y=y, trendline="ols")
-            trendline = fig1.data[1]
-            fig.add_trace(trendline)
-            fig.update_layout(dict(title=self.plot_title))
-
-            fig.write_html(f"{self.original_file.file_output}_plot_{i + 1}.html")
+            Plot(
+                df_close,
+                self.x,
+                self.y,
+                self.plot_title,
+                dst=os.path.dirname(self.original_file.file_output),
+                fname=f"df_plot_{i + 1}",  # TODO: must be user o decides name
+            ).create()
 
     def save(self, df_loop, name):
         return df_loop.to_excel(f"{self.original_file.folder_dst}/df_loop_{name}.xlsx")
 
 
 class Plot:
-    pass
+    def __init__(
+        self, data, x_axis, y_axis, title, *, dst=None, fname="dataframe", output="html",
+    ):
+        self.data = data
+        self.x_axis = x_axis
+        self.y_axis = y_axis
+        self.title = title
+        self.output = output
+        self.fname = fname
+        self.dst = dst
+
+    def create(self):
+        print("Creating plots")
+        x = self.data[self.x_axis]
+        y = self.data[self.y_axis]
+        fig = go.Figure()
+        fig.add_trace(
+            go.Scatter(
+                x=x, y=y, name=self.title, line=dict(color="red", width=1), showlegend=True,
+            )
+        )
+        fig1 = px.scatter(self.data, x=x, y=y, trendline="ols")
+        trendline = fig1.data[1]
+        fig.add_trace(trendline)
+        fig.update_layout(dict(title=self.title))
+
+        fig.write_html(f"{self.dst}/{self.fname}.{self.output}")
 
 
 ControlFile = ExperimentCycle

scripts/stats.py

@@ -159,9 +159,10 @@ class Control(ResumeDataFrame):
 
     def get_bank(self):
         self.generate_resume(0)
-        self.values.append(self.resume_df["CH 1 MO2 [mgO2/hr]"].sum())
+        self.values.append(
+            self.resume_df["CH 1 MO2 [mgO2/hr]"].sum()
+            / len(self.resume_df["CH 1 MO2 [mgO2/hr]"])
+        )
 
     def calculate_blank(self):
-
-        print("total", sum(self.values) / len(self.values))
         return sum(self.values) / len(self.values)

scripts/utils.py

@@ -165,8 +165,6 @@ def string_to_int(value):
         except ValueError:
             return value.strip()
 
-    print(f"{new_config=}")
-
     config_keys = {
         "experiment_file_config": {},
         "file_cycle_config": {},

templates/app.html

@@ -9,12 +9,18 @@
     <div class="row mb-2">
       <div id="status" class="col-md-1">Estat: </div>
       <div id="led" class="col-md-1 d-none">
-        <div id="available" class="led-green"></div>
-        <div id="busy" class="led-red"></div>
-        <div id="stand_by" class="led-yellow"></div>
+        <div id="available" class="led-green" title="Bomba parada"></div>
+        <div id="busy" class="led-red" title="Bomba en funcionament"></div>
+        <div id="stand_by" class="led-yellow" title="Bomba en pausa"></div>
       </div>
       <div class="col-md-4 d-block"></div>
-      <div class="col-md-6" id="timer"></div>
+      <div class="col-md-6">
+        <div class="row">
+          <div class="col-md" id="timer"></div>
+          <div class="col-md" id="auto_run_since"></div>
+          <div class="col-md" id="total_loops"></div>
+        </div>
+      </div>
     </div>
 
     <div class="row mt-3">
@@ -51,19 +57,19 @@
                       data-placement="top"><img class="info-icon" src="{{ url_for('static', filename='img/info.png') }}"></a>
                   </label>
 
-                  <input type="number" class="form-control" name="flush" id="flush" aria-describedby="flushHelp" placeholder="Minuts" value="{{ flush }}" required>
+                  <input type="number" min="0" class="form-control" name="flush" id="flush" aria-describedby="flushHelp" placeholder="Minuts" value="{{ flush }}" required>
                 </div>
                 <div class="col-4">
                   <label for="wait">Wait
                     <a class="ml-3" type="button" data-toggle="popover" data-content="Quant de temps ha de funcionar la bomba per cicle. El valor és de minuts" data-placement="top"><img class="info-icon" src="{{ url_for('static', filename='img/info.png') }}"></a>
                   </label>
-                  <input type="number" class="form-control" name="wait" id="wait" aria-describedby="waitHelp" placeholder="Minuts" value="{{ wait }}" required>
+                  <input type="number" min="0" class="form-control" name="wait" id="wait" aria-describedby="waitHelp" placeholder="Minuts" value="{{ wait }}" required>
                 </div>
                 <div class="col-4">
                   <label for="close">Close
                     <a class="ml-3" type="button" data-toggle="popover" data-content="Quant de temps ha de funcionar la bomba per cicle. El valor és de minuts" data-placement="top"><img class="info-icon" src="{{ url_for('static', filename='img/info.png') }}"></a>
                   </label>
-                  <input type="number" class="form-control" name="close" id="close" aria-describedby="closeHelp" placeholder="Minuts" value="{{ close }}" required>
+                  <input type="number" min="0" class="form-control" name="close" id="close" aria-describedby="closeHelp" placeholder="Minuts" value="{{ close }}" required>
                 </div>
               </div>
               <div class="row mt-4">

templates/base.html

@@ -104,6 +104,8 @@
           $("#stop_auto, #stop_manual").addClass("deactivated")
         }
         $("#led").removeClass("d-none")
+        $("#auto_run_since").text("Started at': " + system_info.auto_run_since);
+        $("#total_loops").text("Total de 'loops': " + system_info.total_loops);
 
         if (system_info.generating_files) {
           $("#submit_files").addClass("deactivated")
@@ -125,6 +127,8 @@
         if (system_info.running && system_info.run_auto) {
           ledRed();
           countDown(system_info.cycle_ends_in);
+          $("#auto_run_since").text("Started at': " + system_info.auto_run_since);
+          $("#total_loops").text("Total de 'loops': " + system_info.total_loops);
         } else if (system_info.run_auto && !system_info.running) {
           ledYellow();
           countDown(system_info.next_cycle_at);
@@ -149,6 +153,7 @@
       $('[data-toggle="popover"]').popover()
     })
   </script>
+  {% block scripts %}{% endblock %}
 
 </body>