diff --git a/maestro2_pipeline.py b/maestro2_pipeline.py index bdf50fa..11f233f 100644 --- a/maestro2_pipeline.py +++ b/maestro2_pipeline.py @@ -195,13 +195,12 @@ def pipeline(study_id: str): # sourcery skip: low-code-quality should_process = file_processor.file_should_process(path, input_last_modified) if not should_process: - logger.time(time_estimator.step()) - logger.debug( f"The file {path} has not been modified since the last time it was processed", ) logger.debug(f"Skipping {path} - File has not been modified") + logger.time(time_estimator.step()) continue file_processor.add_entry(path, input_last_modified) @@ -325,26 +324,26 @@ def pipeline(study_id: str): # sourcery skip: low-code-quality logger.debug("Formatting files and generating metadata") - for device_folder in device_list: - file_list = imaging_utils.get_filtered_file_names(device_folder) + try: + for device_folder in device_list: + file_list = imaging_utils.get_filtered_file_names(device_folder) - try: for file in file_list: if full_file_path := imaging_utils.format_file( file, destination_folder ): maestro2_instance.metadata(full_file_path, metadata_folder) - except Exception: - file_item["format_error"] = True - logger.error(f"Failed to format {file_name}") + except Exception: + file_item["format_error"] = True + logger.error(f"Failed to format {file_name}") - error_exception = "".join(format_exc().splitlines()) + error_exception = "".join(format_exc().splitlines()) - logger.error(error_exception) - file_processor.append_errors(error_exception, path) + logger.error(error_exception) + file_processor.append_errors(error_exception, path) - logger.time(time_estimator.step()) - continue + logger.time(time_estimator.step()) + continue logger.info(f"Formatted {file_name}") diff --git a/optomed_pipeline.py b/optomed_pipeline.py index 2b641af..515bcbf 100644 --- a/optomed_pipeline.py +++ b/optomed_pipeline.py @@ -266,11 +266,10 @@ def pipeline(study_id: str): # sourcery skip: low-code-quality filelist = imaging_utils.get_filtered_file_names(folder) for file in filelist: - full_file_path = imaging_utils.format_file( + if full_file_path := imaging_utils.format_file( file, destination_folder - ) - - optomed_instance.metadata(full_file_path, metadata_folder) + ): + optomed_instance.metadata(full_file_path, metadata_folder) except Exception: logger.error(f"Failed to format {file_name}") @@ -320,12 +319,6 @@ def pipeline(study_id: str): # sourcery skip: low-code-quality file_path=output_file_path ) - # Check if the file already exists. If it does, throw an exception - if output_file_client.exists(): - raise Exception( - f"File {output_file_path} already exists. Throwing exception" - ) - with open(full_file_path, "rb") as f: output_file_client.upload_data(f, overwrite=True) @@ -373,12 +366,6 @@ def pipeline(study_id: str): # sourcery skip: low-code-quality f"Uploading {full_file_path} to {processed_metadata_output_folder}" ) - # Check if the file already exists in the output folder - if output_file_client.exists(): - raise Exception( - f"File {output_file_path} already exists. Throwing exception" - ) - with open(full_file_path, "rb") as f: output_file_client.upload_data(f, overwrite=True) diff --git a/spectralis_pipeline.py b/spectralis_pipeline.py index ab4a56e..4acc79e 100644 --- a/spectralis_pipeline.py +++ b/spectralis_pipeline.py @@ -304,9 +304,12 @@ def pipeline( filelist = imaging_utils.get_filtered_file_names(folder) for file in filelist: - full_file_path = imaging_utils.format_file(file, step4_folder) - - spectralis_instance.metadata(full_file_path, metadata_folder) + if full_file_path := imaging_utils.format_file( + file, step4_folder + ): + spectralis_instance.metadata( + full_file_path, metadata_folder + ) except Exception: logger.error(f"Failed to format {file_name}") @@ -352,12 +355,6 @@ def pipeline( output_file_path ) - # Check if the file already exists. If it does, throw an exception - if output_file_client.exists(): - raise Exception( - f"File {output_file_path} already exists. Throwing exception" - ) - with open(full_file_path, "rb") as f: output_file_client.upload_data(f, overwrite=True) logger.info(f"Uploaded {combined_file_name}") @@ -401,12 +398,6 @@ def pipeline( f"Uploading {full_file_path} to {processed_metadata_output_folder}" ) - # Check if the file already exists in the output folder - if output_file_client.exists(): - raise Exception( - f"File {output_file_path} already exists. Throwing exception" - ) - with open(full_file_path, "rb") as f: output_file_client.upload_data(f, overwrite=True) diff --git a/triton_pipeline.py b/triton_pipeline.py index 01f61e7..ea36d82 100644 --- a/triton_pipeline.py +++ b/triton_pipeline.py @@ -321,28 +321,26 @@ def pipeline(study_id: str): # sourcery skip: low-code-quality logger.debug("Formatting files and generating metadata") - for device_folder in device_list: - file_list = imaging_utils.get_filtered_file_names(device_folder) - - for file_name in file_list: - - try: - full_file_path = imaging_utils.format_file( - file_name, destination_folder - ) - - triton_instance.metadata(full_file_path, metadata_folder) - except Exception: - file_item["format_error"] = True - logger.error(f"Failed to format {file_name}") + try: + for device_folder in device_list: + file_list = imaging_utils.get_filtered_file_names(device_folder) + + for file in file_list: + if full_file_path := imaging_utils.format_file( + file, destination_folder + ): + triton_instance.metadata(full_file_path, metadata_folder) + except Exception: + file_item["format_error"] = True + logger.error(f"Failed to format {file_name}") - error_exception = "".join(format_exc().splitlines()) + error_exception = "".join(format_exc().splitlines()) - logger.error(error_exception) - file_processor.append_errors(error_exception, path) + logger.error(error_exception) + file_processor.append_errors(error_exception, path) - logger.time(time_estimator.step()) - continue + logger.time(time_estimator.step()) + continue logger.info(f"Formatted {file_name}") file_item["processed"] = True @@ -380,12 +378,6 @@ def pipeline(study_id: str): # sourcery skip: low-code-quality file_path=output_file_path ) - # Check if the file already exists. If it does, throw an exception - if output_file_client.exists(): - raise Exception( - f"File {output_file_path} already exists. Throwing exception" - ) - with open(f"{full_file_path}", "rb") as data: output_file_client.upload_data(data, overwrite=True) @@ -428,11 +420,6 @@ def pipeline(study_id: str): # sourcery skip: low-code-quality output_file_client = file_system_client.get_file_client( file_path=output_file_path ) - # Check if the file already exists in the output folder - if output_file_client.exists(): - raise Exception( - f"File {output_file_path} already exists. Throwing exception" - ) with open(full_file_path, "rb") as f: output_file_client.upload_data(f, overwrite=True)