🐛 fix: update workflows

AI-READI · Sep 20, 2024 · 1935e90 · 1935e90
1 parent ca6d3d3
commit 1935e90
Show file tree

Hide file tree

Showing 4 changed files with 38 additions and 74 deletions.
diff --git a/maestro2_pipeline.py b/maestro2_pipeline.py
@@ -195,13 +195,12 @@ def pipeline(study_id: str):  # sourcery skip: low-code-quality
         should_process = file_processor.file_should_process(path, input_last_modified)
 
         if not should_process:
-            logger.time(time_estimator.step())
-
             logger.debug(
                 f"The file {path} has not been modified since the last time it was processed",
             )
             logger.debug(f"Skipping {path} - File has not been modified")
 
+            logger.time(time_estimator.step())
             continue
 
         file_processor.add_entry(path, input_last_modified)
@@ -325,26 +324,26 @@ def pipeline(study_id: str):  # sourcery skip: low-code-quality
 
             logger.debug("Formatting files and generating metadata")
 
-            for device_folder in device_list:
-                file_list = imaging_utils.get_filtered_file_names(device_folder)
+            try:
+                for device_folder in device_list:
+                    file_list = imaging_utils.get_filtered_file_names(device_folder)
 
-                try:
                     for file in file_list:
                         if full_file_path := imaging_utils.format_file(
                             file, destination_folder
                         ):
                             maestro2_instance.metadata(full_file_path, metadata_folder)
-                except Exception:
-                    file_item["format_error"] = True
-                    logger.error(f"Failed to format {file_name}")
+            except Exception:
+                file_item["format_error"] = True
+                logger.error(f"Failed to format {file_name}")
 
-                    error_exception = "".join(format_exc().splitlines())
+                error_exception = "".join(format_exc().splitlines())
 
-                    logger.error(error_exception)
-                    file_processor.append_errors(error_exception, path)
+                logger.error(error_exception)
+                file_processor.append_errors(error_exception, path)
 
-                    logger.time(time_estimator.step())
-                    continue
+                logger.time(time_estimator.step())
+                continue
 
             logger.info(f"Formatted {file_name}")
 

diff --git a/optomed_pipeline.py b/optomed_pipeline.py
@@ -266,11 +266,10 @@ def pipeline(study_id: str):  # sourcery skip: low-code-quality
                     filelist = imaging_utils.get_filtered_file_names(folder)
 
                     for file in filelist:
-                        full_file_path = imaging_utils.format_file(
+                        if full_file_path := imaging_utils.format_file(
                             file, destination_folder
-                        )
-
-                        optomed_instance.metadata(full_file_path, metadata_folder)
+                        ):
+                            optomed_instance.metadata(full_file_path, metadata_folder)
             except Exception:
                 logger.error(f"Failed to format {file_name}")
 
@@ -320,12 +319,6 @@ def pipeline(study_id: str):  # sourcery skip: low-code-quality
                             file_path=output_file_path
                         )
 
-                        # Check if the file already exists. If it does, throw an exception
-                        if output_file_client.exists():
-                            raise Exception(
-                                f"File {output_file_path} already exists. Throwing exception"
-                            )
-
                         with open(full_file_path, "rb") as f:
                             output_file_client.upload_data(f, overwrite=True)
 
@@ -373,12 +366,6 @@ def pipeline(study_id: str):  # sourcery skip: low-code-quality
                             f"Uploading {full_file_path} to {processed_metadata_output_folder}"
                         )
 
-                        # Check if the file already exists in the output folder
-                        if output_file_client.exists():
-                            raise Exception(
-                                f"File {output_file_path} already exists. Throwing exception"
-                            )
-
                         with open(full_file_path, "rb") as f:
                             output_file_client.upload_data(f, overwrite=True)
 

diff --git a/spectralis_pipeline.py b/spectralis_pipeline.py
@@ -304,9 +304,12 @@ def pipeline(
                     filelist = imaging_utils.get_filtered_file_names(folder)
 
                     for file in filelist:
-                        full_file_path = imaging_utils.format_file(file, step4_folder)
-
-                        spectralis_instance.metadata(full_file_path, metadata_folder)
+                        if full_file_path := imaging_utils.format_file(
+                            file, step4_folder
+                        ):
+                            spectralis_instance.metadata(
+                                full_file_path, metadata_folder
+                            )
             except Exception:
                 logger.error(f"Failed to format {file_name}")
 
@@ -352,12 +355,6 @@ def pipeline(
                             output_file_path
                         )
 
-                        # Check if the file already exists. If it does, throw an exception
-                        if output_file_client.exists():
-                            raise Exception(
-                                f"File {output_file_path} already exists. Throwing exception"
-                            )
-
                         with open(full_file_path, "rb") as f:
                             output_file_client.upload_data(f, overwrite=True)
                             logger.info(f"Uploaded {combined_file_name}")
@@ -401,12 +398,6 @@ def pipeline(
                             f"Uploading {full_file_path} to {processed_metadata_output_folder}"
                         )
 
-                        # Check if the file already exists in the output folder
-                        if output_file_client.exists():
-                            raise Exception(
-                                f"File {output_file_path} already exists. Throwing exception"
-                            )
-
                         with open(full_file_path, "rb") as f:
                             output_file_client.upload_data(f, overwrite=True)
 

diff --git a/triton_pipeline.py b/triton_pipeline.py
@@ -321,28 +321,26 @@ def pipeline(study_id: str):  # sourcery skip: low-code-quality
 
             logger.debug("Formatting files and generating metadata")
 
-            for device_folder in device_list:
-                file_list = imaging_utils.get_filtered_file_names(device_folder)
-
-                for file_name in file_list:
-
-                    try:
-                        full_file_path = imaging_utils.format_file(
-                            file_name, destination_folder
-                        )
-
-                        triton_instance.metadata(full_file_path, metadata_folder)
-                    except Exception:
-                        file_item["format_error"] = True
-                        logger.error(f"Failed to format {file_name}")
+            try:
+                for device_folder in device_list:
+                    file_list = imaging_utils.get_filtered_file_names(device_folder)
+
+                    for file in file_list:
+                        if full_file_path := imaging_utils.format_file(
+                            file, destination_folder
+                        ):
+                            triton_instance.metadata(full_file_path, metadata_folder)
+            except Exception:
+                file_item["format_error"] = True
+                logger.error(f"Failed to format {file_name}")
 
-                        error_exception = "".join(format_exc().splitlines())
+                error_exception = "".join(format_exc().splitlines())
 
-                        logger.error(error_exception)
-                        file_processor.append_errors(error_exception, path)
+                logger.error(error_exception)
+                file_processor.append_errors(error_exception, path)
 
-                        logger.time(time_estimator.step())
-                        continue
+                logger.time(time_estimator.step())
+                continue
 
             logger.info(f"Formatted {file_name}")
             file_item["processed"] = True
@@ -380,12 +378,6 @@ def pipeline(study_id: str):  # sourcery skip: low-code-quality
                             file_path=output_file_path
                         )
 
-                        # Check if the file already exists. If it does, throw an exception
-                        if output_file_client.exists():
-                            raise Exception(
-                                f"File {output_file_path} already exists. Throwing exception"
-                            )
-
                         with open(f"{full_file_path}", "rb") as data:
                             output_file_client.upload_data(data, overwrite=True)
 
@@ -428,11 +420,6 @@ def pipeline(study_id: str):  # sourcery skip: low-code-quality
                         output_file_client = file_system_client.get_file_client(
                             file_path=output_file_path
                         )
-                        # Check if the file already exists in the output folder
-                        if output_file_client.exists():
-                            raise Exception(
-                                f"File {output_file_path} already exists. Throwing exception"
-                            )
 
                         with open(full_file_path, "rb") as f:
                             output_file_client.upload_data(f, overwrite=True)