diff --git a/docs/generator.html b/docs/generator.html new file mode 100644 index 0000000..d2211ef --- /dev/null +++ b/docs/generator.html @@ -0,0 +1,1179 @@ + + + + + + +lumos.generator API documentation + + + + + + + + + + + +
+
+
+

Module lumos.generator

+
+
+

Main functions to generate platemaps with lumos.

+
+ +Expand source code + +
#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+'''
+Main functions to generate platemaps with lumos.
+'''
+
+import math
+import multiprocessing
+import os
+from pathlib import Path
+import platform
+import pandas as pd
+from shutil import copyfile
+import shutil
+from . import toolbox
+from . import logger
+from . import parameters
+import cv2
+from tqdm import tqdm
+import numpy as np
+
+
+def generate_plate_image_for_channel(
+    plate_input_path_string,
+    plate_name,
+    channel_to_render,
+    channel_label,
+    temp_folder,
+    keep_temp_files,
+):
+    '''
+    Generates an image of a cellpainting plate for a specific channel.
+
+            Parameters:
+                    plate_images_folder_path (Path): The path to the folder where the images of the plate are stored.
+                    plate_name (string): Name of the plate.
+                    channel_to_render (string): The cellpainting channel to render.
+                    channel_label (string): The label describing the channel type.
+                    temp_folder_path (Path): The folder where temporary data can be stored.
+
+            Returns:
+                    8-bit cv2 image
+    '''
+
+    # define a temp folder for the run
+    temp_folder = temp_folder + "/lumos-tmpgen-" + plate_name + channel_to_render
+
+    # remove temp dir if existing
+    if not keep_temp_files:
+        logger.debug("Purge temporary folder before plate generation")
+        shutil.rmtree(temp_folder, ignore_errors=True)
+
+    # create the temporary directory structure to work on images
+    try:
+        os.mkdir(temp_folder)
+        os.mkdir(temp_folder + "/wells")
+    except FileExistsError:
+        pass
+
+    # read the plate input path
+    plate_input_path = Path(plate_input_path_string)
+
+    # get the files from the plate folder, for the targeted channel
+    images_full_path_list = list(
+        Path(plate_input_path).glob("*" + channel_to_render + ".tif")
+    )
+
+    # check that we get 2304 images for a 384 well image
+    try:
+        assert len(images_full_path_list) == 2304
+    except AssertionError:
+        logger.p_print(
+            "The plate does not have the exact image count: expected 2304, got "
+            + str(len(images_full_path_list))
+        )
+        logger.warning(
+            "The plate does not have the exact image count: expected 2304, got "
+            + str(len(images_full_path_list))
+        )
+
+    logger.info(
+        "Start plate image generation for channel: "
+        + str(channel_to_render)
+        + " - "
+        + str(channel_label)
+    )
+
+    # get the filenames list
+    images_full_path_list.sort()
+    images_filename_list = [str(x.name) for x in images_full_path_list]
+
+    # get the well list
+    image_well_list = [x.split("_")[1].split("_T")[0] for x in images_filename_list]
+
+    # get the siteid list (sitesid from 1 to 6)
+    image_site_list = [
+        x.split("_T0001F")[1].split("L")[0] for x in images_filename_list
+    ]
+    image_site_list_int = [int(x) for x in image_site_list]
+
+    # zip all in a data structure
+    image_data_zip = zip(
+        image_well_list,
+        image_site_list_int,
+        images_filename_list,
+        images_full_path_list,
+    )
+
+    # convert the zip into dataframe
+    data_df = pd.DataFrame(
+        list(image_data_zip), columns=["well", "site", "filename", "fullpath"]
+    )
+
+    # get the theoretical well list for 384 well plate
+    well_theoretical_list = [
+        l + str(r).zfill(2) for l in "ABCDEFGHIJKLMNOP" for r in range(1, 25)       # e.g. "A01"
+    ]
+    well_site_theoretical_list = [
+        [x, r] for x in well_theoretical_list for r in range(1, 7)                  # e.g. ["A01", 1] .. ["A01", 6]
+    ]
+
+    # create the theoretical well dataframe
+    theoretical_data_df = pd.DataFrame(
+        well_site_theoretical_list, columns=["well", "site"]
+    )
+
+    # join the real wells with the theoric ones
+    data_df_joined = theoretical_data_df.merge(
+        data_df,
+        left_on=["well", "site"],
+        right_on=["well", "site"],
+        how="left",
+    )
+
+    # log if there is a delta between theory and actual plate wells
+    delta = set(well_theoretical_list) - set(image_well_list)
+    logger.debug("Well Delta " + str(delta))
+
+    # get the site images and store them locally
+    logger.info("Copying sources images in temp folder..")
+
+    copyprogressbar = tqdm(
+        data_df_joined.iterrows(),
+        total=len(data_df_joined),
+        desc="Download images to temp",
+        unit="images",
+        colour="blue" if platform.system() == 'Windows' else "#006464",
+        leave=True,
+        disable=logger._is_in_parallel,
+    )
+    for _, current_image in copyprogressbar:
+
+        # do not copy if temp file already exists, or if source file doesn't exists
+        if not os.path.isfile(temp_folder + "/" + str(current_image["filename"])):
+            try:
+                copyfile(
+                    current_image["fullpath"],
+                    temp_folder + "/" + str(current_image["filename"]),
+                )
+            except TypeError:
+                # this is thrown when the source file does not exist, or when copyfile() fails
+                logger.warning(
+                    "TypeError: from "
+                    + str(current_image["fullpath"])
+                    + " to "
+                    + str(temp_folder)
+                    + "/"
+                    + str(current_image["filename"])
+                )
+        else:
+            logger.debug(
+                "File already exists in temp folder: "
+                + temp_folder + "/" + str(current_image["filename"])
+            )
+
+    logger.info("Copying sources images in temp folder..Done")
+
+    # get the list of all the wells
+    # We first convert to a set to remove redundant wells (duplicate data because each is represented 6 times, one per site)
+    well_list = list(set(data_df_joined["well"]))
+    well_list.sort()
+
+    logger.info("Generating well images and storing them in temp dir..")
+
+    # generate one image per well by concatenation of image sites
+    wellprogressbar = tqdm(
+        well_list,
+        unit="wells",
+        colour="magenta" if platform.system() == 'Windows' else "#6464a0",
+        leave=True,
+        disable=logger._is_in_parallel,
+    )
+    for current_well in wellprogressbar:
+        wellprogressbar.set_description("Processing well %s" % current_well)
+
+        # get the 6 images metadata of the well
+        current_wells_df = data_df_joined.loc[data_df_joined["well"] == current_well]
+
+        # load 6 wells into an image list (if image cannot be opened, e.g. if it is missing or corrupted, replace with a placeholder image)
+        image_list = []
+        for current_site in range(1, 7):
+            img = toolbox.load_site_image(current_site, current_wells_df, temp_folder)
+            try:
+                # resize the image first to reduce computations
+                img = cv2.resize(
+                    src=img,
+                    dsize=None,
+                    fx=parameters.rescale_ratio,
+                    fy=parameters.rescale_ratio,
+                    interpolation=cv2.INTER_CUBIC,
+                )
+                # normalize the intensity of each channel by a specific coefficient
+                img = img * parameters.channel_coefficients[channel_to_render]
+                # convert to 8 bit
+                img = img / 256
+                img = img.astype("uint8")
+            except:
+                # create placeholder image when error
+                img = np.full(
+                    shape=(int(1000*parameters.rescale_ratio), int(1000*parameters.rescale_ratio), 1),
+                    fill_value=parameters.placeholder_background_intensity,
+                    dtype=np.uint8
+                )
+                img = toolbox.draw_markers(img, parameters.placeholder_markers_intensity)
+                logger.warning("Missing or corrupted file in well " + current_well + " (site " + str(current_site) + ")")
+
+            image_list.append(img)
+
+        # concatenate horizontally and vertically
+        sites_row1 = cv2.hconcat(
+            [image_list[0], image_list[1], image_list[2]]
+        )
+        sites_row2 = cv2.hconcat(
+            [image_list[3], image_list[4], image_list[5]]
+        )
+        all_sites_image = cv2.vconcat([sites_row1, sites_row2])
+
+        # add well id on image
+        text = current_well + " " + channel_label
+        font = cv2.FONT_HERSHEY_SIMPLEX
+        cv2.putText(
+            all_sites_image,
+            text,
+            (math.ceil(25*parameters.rescale_ratio), math.ceil(125*parameters.rescale_ratio)),
+            font,
+            4*parameters.rescale_ratio,
+            (192, 192, 192),
+            math.ceil(8*parameters.rescale_ratio),
+            cv2.INTER_AREA,
+        )
+
+        # add well marks on borders
+        image_shape = all_sites_image.shape
+        cv2.rectangle(
+            all_sites_image,
+            (0, 0),
+            (image_shape[1], image_shape[0]),
+            color=(192, 192, 192),
+            thickness=1,
+        )
+
+        # save the image in the temp folder
+        cv2.imwrite(
+            temp_folder + "/wells/well-" + str(current_well) + ".png",
+            all_sites_image,
+        )
+
+    logger.info("Generating well images and storing them in temp dir..Done")
+
+    # load all well images and store images in memory into a list
+    logger.p_print("Combining well images into final channel image..")
+    logger.info("Loading well images from temp dir..")
+
+    image_well_data = []
+    for current_well in list(well_list):
+        well_image = toolbox.load_well_image(
+            current_well,
+            temp_folder + "/wells",
+        )
+        image_well_data.append(well_image)
+
+    logger.info("Loading well images from temp dir..Done")
+
+    # concatenate all the well images into horizontal stripes (1 per row)
+    logger.info("Concatenating well images into a plate..")
+
+    image_row_data = []
+    for current_plate_row in range(1, 17):
+
+        # concatenate horizontally and vertically
+        well_start_id = ((current_plate_row - 1) * 24) + 0
+        well_end_id = current_plate_row * 24
+        sites_row = cv2.hconcat(image_well_data[well_start_id:well_end_id])
+        image_row_data.append(sites_row)
+
+    # concatenate all the stripes into 1 image
+    plate_image = cv2.vconcat(image_row_data)
+
+    logger.info("Concatenating well images into a plate..Done")
+
+    # purge temp files
+    if not keep_temp_files:
+        logger.debug("Purge temporary folder after generation")
+        shutil.rmtree(temp_folder, ignore_errors=True)
+
+    return plate_image
+
+
+def render_single_channel_plateview(
+    source_path, plate_name, channel_to_render, channel_label, output_path, temp_folder_path, keep_temp_files
+):
+    '''
+    Renders 1 image for a specific channel of a plate.
+
+            Parameters:
+                    source_path (Path): The path to the folder where the images of the plate are stored.
+                    plate_name (string): Name of the plate.
+                    channel_to_render (string): The name of the channel to render.
+                    channel_label (string): The label describing the channel type.
+                    output_path (Path): The folder where to save the generated image.
+                    temp_folder_path (Path): The folder where temporary data can be stored.
+                    keep_temp_files (bool): [dev] Whether or not the temporary files should be kept between runs.
+
+            Returns:
+                    True (in case of success)
+    '''
+
+    # generate cv2 image for the channel
+    plate_image = generate_plate_image_for_channel(
+        source_path,
+        plate_name,
+        channel_to_render,
+        channel_label,
+        temp_folder_path,
+        keep_temp_files
+    )
+    logger.p_print(" -> Generated image of size: " + str(plate_image.shape))
+
+    # save image
+    plate_image_path = (
+        output_path
+        + "/"
+        + plate_name
+        + "-"
+        + str(channel_to_render)
+        + "-"
+        + str(parameters.channel_coefficients[channel_to_render])
+        + ".jpg"
+    )
+    cv2.imwrite(plate_image_path, plate_image)
+    logger.p_print(" -> Saved as " + plate_image_path)
+
+    return
+
+
+def render_single_plate_plateview(
+    source_path,
+    plate_name,
+    channel_list,
+    output_path,
+    temp_folder_path,
+    keep_temp_files
+):
+    '''
+    Renders 1 image per channel for a specific plate.
+
+            Parameters:
+                    source_path (Path): The path to the folder where the images of the plate are stored.
+                    plate_name (string): Name of the plate.
+                    channel_list (string list): The list of the channels to render.
+                    output_path (Path): The folder where to save the generated image.
+                    temp_folder_path (Path): The folder where temporary data can be stored.
+                    keep_temp_files (bool): [dev] Whether or not the temporary files should be kept between runs.
+
+            Returns:
+                    True (in case of success)
+    '''
+
+    for current_channel in tqdm(
+        channel_list,
+        desc="Render plate channels",
+        unit="channel",
+        colour="green" if platform.system() == 'Windows' else "#00ff00",
+    ):
+        # get the current channel's label
+        channel_label = parameters.cellplainting_channels_dict[current_channel]
+
+        logger.p_print(os.linesep)
+        logger.p_print("Generate " + current_channel + " - " + channel_label + os.linesep)
+
+        render_single_channel_plateview(
+            source_path,
+            plate_name,
+            current_channel,
+            channel_label,
+            output_path,
+            temp_folder_path,
+            keep_temp_files
+        )
+
+    return
+
+
+def render_single_plate_plateview_parallelism(
+    source_path,
+    plate_name,
+    channel_list,
+    output_path,
+    temp_folder_path,
+    parallelism,
+    keep_temp_files
+):
+    '''
+    Renders, in parallel, 1 image per channel for a specific plate.
+
+            Parameters:
+                    source_path (Path): The path to the folder where the images of the plate are stored.
+                    plate_name (string): Name of the plate.
+                    channel_list (string list): The list of the channels to render.
+                    output_path (Path): The folder where to save the generated image.
+                    temp_folder_path (Path): The folder where temporary data can be stored.
+                    parallelism (int): On how many CPU cores should the computation be spread.
+                    keep_temp_files (bool): [dev] Whether or not the temporary files should be kept between runs.
+
+            Returns:
+                    True (in case of success)
+    '''
+
+    n_cores = min(parallelism, multiprocessing.cpu_count())
+    pool = multiprocessing.Pool(n_cores)
+
+    try:
+        for current_channel in channel_list:
+            # get the current channel's label
+            channel_label = parameters.cellplainting_channels_dict[current_channel]
+
+            pool.apply_async(render_single_channel_plateview, args=(
+                source_path,
+                plate_name,
+                current_channel,
+                channel_label,
+                output_path,
+                temp_folder_path,
+                keep_temp_files
+            ))
+
+        pool.close()
+        pool.join()
+
+    except KeyboardInterrupt:
+        # does not work: this is an issue with the multiprocessing library
+        pool.terminate()
+        pool.join()
+
+    return
+
+
+def render_single_run_plateview(
+    source_folder_dict,
+    channel_list,
+    output_path,
+    temp_folder_path,
+    parallelism,
+    keep_temp_files
+):
+    '''
+    Renders images for all plates of a run. Compatible with parallelism.
+
+            Parameters:
+                    source_folder_dict (dict): A dictionary of the name of the plates and their respective path.
+                    channel_list (string list): The list of the channels to render for all plates.
+                    output_path (Path): The folder where to save the generated image.
+                    temp_folder_path (Path): The folder where temporary data can be stored.
+                    parallelism (int): On how many CPU cores should the computation be spread.
+                    keep_temp_files (bool): [dev] Whether or not the temporary files should be kept between runs.
+
+            Returns:
+                    True (in case of success)
+    '''
+    runprogressbar = tqdm(
+        source_folder_dict.keys(),
+        total=len(source_folder_dict),
+        desc="Run progress",
+        unit="plates",
+        colour='cyan' if platform.system() == 'Windows' else "#0AAFAF",
+        leave=True,
+    )
+    for current_plate in runprogressbar:
+        # render all the channels of the plate
+        if parallelism == 1:
+            render_single_plate_plateview(
+                source_folder_dict[current_plate],
+                current_plate,
+                channel_list,
+                output_path,
+                temp_folder_path,
+                keep_temp_files,
+            )
+        else:
+            render_single_plate_plateview_parallelism(
+                source_folder_dict[current_plate],
+                current_plate,
+                channel_list,
+                output_path,
+                temp_folder_path,
+                parallelism,
+                keep_temp_files,
+            )
+
+    print(os.linesep + os.linesep + "Run completed!")
+    print(str(len(source_folder_dict.keys())), "plate(s) have been processed.", os.linesep)
+
+    return
+
+
+
+
+
+
+
+

Functions

+
+
+def generate_plate_image_for_channel(plate_input_path_string, plate_name, channel_to_render, channel_label, temp_folder, keep_temp_files) +
+
+

Generates an image of a cellpainting plate for a specific channel.

+
    Parameters:
+            plate_images_folder_path (Path): The path to the folder where the images of the plate are stored.
+            plate_name (string): Name of the plate.
+            channel_to_render (string): The cellpainting channel to render.
+            channel_label (string): The label describing the channel type.
+            temp_folder_path (Path): The folder where temporary data can be stored.
+
+    Returns:
+            8-bit cv2 image
+
+
+ +Expand source code + +
def generate_plate_image_for_channel(
+    plate_input_path_string,
+    plate_name,
+    channel_to_render,
+    channel_label,
+    temp_folder,
+    keep_temp_files,
+):
+    '''
+    Generates an image of a cellpainting plate for a specific channel.
+
+            Parameters:
+                    plate_images_folder_path (Path): The path to the folder where the images of the plate are stored.
+                    plate_name (string): Name of the plate.
+                    channel_to_render (string): The cellpainting channel to render.
+                    channel_label (string): The label describing the channel type.
+                    temp_folder_path (Path): The folder where temporary data can be stored.
+
+            Returns:
+                    8-bit cv2 image
+    '''
+
+    # define a temp folder for the run
+    temp_folder = temp_folder + "/lumos-tmpgen-" + plate_name + channel_to_render
+
+    # remove temp dir if existing
+    if not keep_temp_files:
+        logger.debug("Purge temporary folder before plate generation")
+        shutil.rmtree(temp_folder, ignore_errors=True)
+
+    # create the temporary directory structure to work on images
+    try:
+        os.mkdir(temp_folder)
+        os.mkdir(temp_folder + "/wells")
+    except FileExistsError:
+        pass
+
+    # read the plate input path
+    plate_input_path = Path(plate_input_path_string)
+
+    # get the files from the plate folder, for the targeted channel
+    images_full_path_list = list(
+        Path(plate_input_path).glob("*" + channel_to_render + ".tif")
+    )
+
+    # check that we get 2304 images for a 384 well image
+    try:
+        assert len(images_full_path_list) == 2304
+    except AssertionError:
+        logger.p_print(
+            "The plate does not have the exact image count: expected 2304, got "
+            + str(len(images_full_path_list))
+        )
+        logger.warning(
+            "The plate does not have the exact image count: expected 2304, got "
+            + str(len(images_full_path_list))
+        )
+
+    logger.info(
+        "Start plate image generation for channel: "
+        + str(channel_to_render)
+        + " - "
+        + str(channel_label)
+    )
+
+    # get the filenames list
+    images_full_path_list.sort()
+    images_filename_list = [str(x.name) for x in images_full_path_list]
+
+    # get the well list
+    image_well_list = [x.split("_")[1].split("_T")[0] for x in images_filename_list]
+
+    # get the siteid list (sitesid from 1 to 6)
+    image_site_list = [
+        x.split("_T0001F")[1].split("L")[0] for x in images_filename_list
+    ]
+    image_site_list_int = [int(x) for x in image_site_list]
+
+    # zip all in a data structure
+    image_data_zip = zip(
+        image_well_list,
+        image_site_list_int,
+        images_filename_list,
+        images_full_path_list,
+    )
+
+    # convert the zip into dataframe
+    data_df = pd.DataFrame(
+        list(image_data_zip), columns=["well", "site", "filename", "fullpath"]
+    )
+
+    # get the theoretical well list for 384 well plate
+    well_theoretical_list = [
+        l + str(r).zfill(2) for l in "ABCDEFGHIJKLMNOP" for r in range(1, 25)       # e.g. "A01"
+    ]
+    well_site_theoretical_list = [
+        [x, r] for x in well_theoretical_list for r in range(1, 7)                  # e.g. ["A01", 1] .. ["A01", 6]
+    ]
+
+    # create the theoretical well dataframe
+    theoretical_data_df = pd.DataFrame(
+        well_site_theoretical_list, columns=["well", "site"]
+    )
+
+    # join the real wells with the theoric ones
+    data_df_joined = theoretical_data_df.merge(
+        data_df,
+        left_on=["well", "site"],
+        right_on=["well", "site"],
+        how="left",
+    )
+
+    # log if there is a delta between theory and actual plate wells
+    delta = set(well_theoretical_list) - set(image_well_list)
+    logger.debug("Well Delta " + str(delta))
+
+    # get the site images and store them locally
+    logger.info("Copying sources images in temp folder..")
+
+    copyprogressbar = tqdm(
+        data_df_joined.iterrows(),
+        total=len(data_df_joined),
+        desc="Download images to temp",
+        unit="images",
+        colour="blue" if platform.system() == 'Windows' else "#006464",
+        leave=True,
+        disable=logger._is_in_parallel,
+    )
+    for _, current_image in copyprogressbar:
+
+        # do not copy if temp file already exists, or if source file doesn't exists
+        if not os.path.isfile(temp_folder + "/" + str(current_image["filename"])):
+            try:
+                copyfile(
+                    current_image["fullpath"],
+                    temp_folder + "/" + str(current_image["filename"]),
+                )
+            except TypeError:
+                # this is thrown when the source file does not exist, or when copyfile() fails
+                logger.warning(
+                    "TypeError: from "
+                    + str(current_image["fullpath"])
+                    + " to "
+                    + str(temp_folder)
+                    + "/"
+                    + str(current_image["filename"])
+                )
+        else:
+            logger.debug(
+                "File already exists in temp folder: "
+                + temp_folder + "/" + str(current_image["filename"])
+            )
+
+    logger.info("Copying sources images in temp folder..Done")
+
+    # get the list of all the wells
+    # We first convert to a set to remove redundant wells (duplicate data because each is represented 6 times, one per site)
+    well_list = list(set(data_df_joined["well"]))
+    well_list.sort()
+
+    logger.info("Generating well images and storing them in temp dir..")
+
+    # generate one image per well by concatenation of image sites
+    wellprogressbar = tqdm(
+        well_list,
+        unit="wells",
+        colour="magenta" if platform.system() == 'Windows' else "#6464a0",
+        leave=True,
+        disable=logger._is_in_parallel,
+    )
+    for current_well in wellprogressbar:
+        wellprogressbar.set_description("Processing well %s" % current_well)
+
+        # get the 6 images metadata of the well
+        current_wells_df = data_df_joined.loc[data_df_joined["well"] == current_well]
+
+        # load 6 wells into an image list (if image cannot be opened, e.g. if it is missing or corrupted, replace with a placeholder image)
+        image_list = []
+        for current_site in range(1, 7):
+            img = toolbox.load_site_image(current_site, current_wells_df, temp_folder)
+            try:
+                # resize the image first to reduce computations
+                img = cv2.resize(
+                    src=img,
+                    dsize=None,
+                    fx=parameters.rescale_ratio,
+                    fy=parameters.rescale_ratio,
+                    interpolation=cv2.INTER_CUBIC,
+                )
+                # normalize the intensity of each channel by a specific coefficient
+                img = img * parameters.channel_coefficients[channel_to_render]
+                # convert to 8 bit
+                img = img / 256
+                img = img.astype("uint8")
+            except:
+                # create placeholder image when error
+                img = np.full(
+                    shape=(int(1000*parameters.rescale_ratio), int(1000*parameters.rescale_ratio), 1),
+                    fill_value=parameters.placeholder_background_intensity,
+                    dtype=np.uint8
+                )
+                img = toolbox.draw_markers(img, parameters.placeholder_markers_intensity)
+                logger.warning("Missing or corrupted file in well " + current_well + " (site " + str(current_site) + ")")
+
+            image_list.append(img)
+
+        # concatenate horizontally and vertically
+        sites_row1 = cv2.hconcat(
+            [image_list[0], image_list[1], image_list[2]]
+        )
+        sites_row2 = cv2.hconcat(
+            [image_list[3], image_list[4], image_list[5]]
+        )
+        all_sites_image = cv2.vconcat([sites_row1, sites_row2])
+
+        # add well id on image
+        text = current_well + " " + channel_label
+        font = cv2.FONT_HERSHEY_SIMPLEX
+        cv2.putText(
+            all_sites_image,
+            text,
+            (math.ceil(25*parameters.rescale_ratio), math.ceil(125*parameters.rescale_ratio)),
+            font,
+            4*parameters.rescale_ratio,
+            (192, 192, 192),
+            math.ceil(8*parameters.rescale_ratio),
+            cv2.INTER_AREA,
+        )
+
+        # add well marks on borders
+        image_shape = all_sites_image.shape
+        cv2.rectangle(
+            all_sites_image,
+            (0, 0),
+            (image_shape[1], image_shape[0]),
+            color=(192, 192, 192),
+            thickness=1,
+        )
+
+        # save the image in the temp folder
+        cv2.imwrite(
+            temp_folder + "/wells/well-" + str(current_well) + ".png",
+            all_sites_image,
+        )
+
+    logger.info("Generating well images and storing them in temp dir..Done")
+
+    # load all well images and store images in memory into a list
+    logger.p_print("Combining well images into final channel image..")
+    logger.info("Loading well images from temp dir..")
+
+    image_well_data = []
+    for current_well in list(well_list):
+        well_image = toolbox.load_well_image(
+            current_well,
+            temp_folder + "/wells",
+        )
+        image_well_data.append(well_image)
+
+    logger.info("Loading well images from temp dir..Done")
+
+    # concatenate all the well images into horizontal stripes (1 per row)
+    logger.info("Concatenating well images into a plate..")
+
+    image_row_data = []
+    for current_plate_row in range(1, 17):
+
+        # concatenate horizontally and vertically
+        well_start_id = ((current_plate_row - 1) * 24) + 0
+        well_end_id = current_plate_row * 24
+        sites_row = cv2.hconcat(image_well_data[well_start_id:well_end_id])
+        image_row_data.append(sites_row)
+
+    # concatenate all the stripes into 1 image
+    plate_image = cv2.vconcat(image_row_data)
+
+    logger.info("Concatenating well images into a plate..Done")
+
+    # purge temp files
+    if not keep_temp_files:
+        logger.debug("Purge temporary folder after generation")
+        shutil.rmtree(temp_folder, ignore_errors=True)
+
+    return plate_image
+
+
+
+def render_single_channel_plateview(source_path, plate_name, channel_to_render, channel_label, output_path, temp_folder_path, keep_temp_files) +
+
+

Renders 1 image for a specific channel of a plate.

+
    Parameters:
+            source_path (Path): The path to the folder where the images of the plate are stored.
+            plate_name (string): Name of the plate.
+            channel_to_render (string): The name of the channel to render.
+            channel_label (string): The label describing the channel type.
+            output_path (Path): The folder where to save the generated image.
+            temp_folder_path (Path): The folder where temporary data can be stored.
+            keep_temp_files (bool): [dev] Whether or not the temporary files should be kept between runs.
+
+    Returns:
+            True (in case of success)
+
+
+ +Expand source code + +
def render_single_channel_plateview(
+    source_path, plate_name, channel_to_render, channel_label, output_path, temp_folder_path, keep_temp_files
+):
+    '''
+    Renders 1 image for a specific channel of a plate.
+
+            Parameters:
+                    source_path (Path): The path to the folder where the images of the plate are stored.
+                    plate_name (string): Name of the plate.
+                    channel_to_render (string): The name of the channel to render.
+                    channel_label (string): The label describing the channel type.
+                    output_path (Path): The folder where to save the generated image.
+                    temp_folder_path (Path): The folder where temporary data can be stored.
+                    keep_temp_files (bool): [dev] Whether or not the temporary files should be kept between runs.
+
+            Returns:
+                    True (in case of success)
+    '''
+
+    # generate cv2 image for the channel
+    plate_image = generate_plate_image_for_channel(
+        source_path,
+        plate_name,
+        channel_to_render,
+        channel_label,
+        temp_folder_path,
+        keep_temp_files
+    )
+    logger.p_print(" -> Generated image of size: " + str(plate_image.shape))
+
+    # save image
+    plate_image_path = (
+        output_path
+        + "/"
+        + plate_name
+        + "-"
+        + str(channel_to_render)
+        + "-"
+        + str(parameters.channel_coefficients[channel_to_render])
+        + ".jpg"
+    )
+    cv2.imwrite(plate_image_path, plate_image)
+    logger.p_print(" -> Saved as " + plate_image_path)
+
+    return
+
+
+
+def render_single_plate_plateview(source_path, plate_name, channel_list, output_path, temp_folder_path, keep_temp_files) +
+
+

Renders 1 image per channel for a specific plate.

+
    Parameters:
+            source_path (Path): The path to the folder where the images of the plate are stored.
+            plate_name (string): Name of the plate.
+            channel_list (string list): The list of the channels to render.
+            output_path (Path): The folder where to save the generated image.
+            temp_folder_path (Path): The folder where temporary data can be stored.
+            keep_temp_files (bool): [dev] Whether or not the temporary files should be kept between runs.
+
+    Returns:
+            True (in case of success)
+
+
+ +Expand source code + +
def render_single_plate_plateview(
+    source_path,
+    plate_name,
+    channel_list,
+    output_path,
+    temp_folder_path,
+    keep_temp_files
+):
+    '''
+    Renders 1 image per channel for a specific plate.
+
+            Parameters:
+                    source_path (Path): The path to the folder where the images of the plate are stored.
+                    plate_name (string): Name of the plate.
+                    channel_list (string list): The list of the channels to render.
+                    output_path (Path): The folder where to save the generated image.
+                    temp_folder_path (Path): The folder where temporary data can be stored.
+                    keep_temp_files (bool): [dev] Whether or not the temporary files should be kept between runs.
+
+            Returns:
+                    True (in case of success)
+    '''
+
+    for current_channel in tqdm(
+        channel_list,
+        desc="Render plate channels",
+        unit="channel",
+        colour="green" if platform.system() == 'Windows' else "#00ff00",
+    ):
+        # get the current channel's label
+        channel_label = parameters.cellplainting_channels_dict[current_channel]
+
+        logger.p_print(os.linesep)
+        logger.p_print("Generate " + current_channel + " - " + channel_label + os.linesep)
+
+        render_single_channel_plateview(
+            source_path,
+            plate_name,
+            current_channel,
+            channel_label,
+            output_path,
+            temp_folder_path,
+            keep_temp_files
+        )
+
+    return
+
+
+
+def render_single_plate_plateview_parallelism(source_path, plate_name, channel_list, output_path, temp_folder_path, parallelism, keep_temp_files) +
+
+

Renders, in parallel, 1 image per channel for a specific plate.

+
    Parameters:
+            source_path (Path): The path to the folder where the images of the plate are stored.
+            plate_name (string): Name of the plate.
+            channel_list (string list): The list of the channels to render.
+            output_path (Path): The folder where to save the generated image.
+            temp_folder_path (Path): The folder where temporary data can be stored.
+            parallelism (int): On how many CPU cores should the computation be spread.
+            keep_temp_files (bool): [dev] Whether or not the temporary files should be kept between runs.
+
+    Returns:
+            True (in case of success)
+
+
+ +Expand source code + +
def render_single_plate_plateview_parallelism(
+    source_path,
+    plate_name,
+    channel_list,
+    output_path,
+    temp_folder_path,
+    parallelism,
+    keep_temp_files
+):
+    '''
+    Renders, in parallel, 1 image per channel for a specific plate.
+
+            Parameters:
+                    source_path (Path): The path to the folder where the images of the plate are stored.
+                    plate_name (string): Name of the plate.
+                    channel_list (string list): The list of the channels to render.
+                    output_path (Path): The folder where to save the generated image.
+                    temp_folder_path (Path): The folder where temporary data can be stored.
+                    parallelism (int): On how many CPU cores should the computation be spread.
+                    keep_temp_files (bool): [dev] Whether or not the temporary files should be kept between runs.
+
+            Returns:
+                    True (in case of success)
+    '''
+
+    n_cores = min(parallelism, multiprocessing.cpu_count())
+    pool = multiprocessing.Pool(n_cores)
+
+    try:
+        for current_channel in channel_list:
+            # get the current channel's label
+            channel_label = parameters.cellplainting_channels_dict[current_channel]
+
+            pool.apply_async(render_single_channel_plateview, args=(
+                source_path,
+                plate_name,
+                current_channel,
+                channel_label,
+                output_path,
+                temp_folder_path,
+                keep_temp_files
+            ))
+
+        pool.close()
+        pool.join()
+
+    except KeyboardInterrupt:
+        # does not work: this is an issue with the multiprocessing library
+        pool.terminate()
+        pool.join()
+
+    return
+
+
+
+def render_single_run_plateview(source_folder_dict, channel_list, output_path, temp_folder_path, parallelism, keep_temp_files) +
+
+

Renders images for all plates of a run. Compatible with parallelism.

+
    Parameters:
+            source_folder_dict (dict): A dictionary of the name of the plates and their respective path.
+            channel_list (string list): The list of the channels to render for all plates.
+            output_path (Path): The folder where to save the generated image.
+            temp_folder_path (Path): The folder where temporary data can be stored.
+            parallelism (int): On how many CPU cores should the computation be spread.
+            keep_temp_files (bool): [dev] Whether or not the temporary files should be kept between runs.
+
+    Returns:
+            True (in case of success)
+
+
+ +Expand source code + +
def render_single_run_plateview(
+    source_folder_dict,
+    channel_list,
+    output_path,
+    temp_folder_path,
+    parallelism,
+    keep_temp_files
+):
+    '''
+    Renders images for all plates of a run. Compatible with parallelism.
+
+            Parameters:
+                    source_folder_dict (dict): A dictionary of the name of the plates and their respective path.
+                    channel_list (string list): The list of the channels to render for all plates.
+                    output_path (Path): The folder where to save the generated image.
+                    temp_folder_path (Path): The folder where temporary data can be stored.
+                    parallelism (int): On how many CPU cores should the computation be spread.
+                    keep_temp_files (bool): [dev] Whether or not the temporary files should be kept between runs.
+
+            Returns:
+                    True (in case of success)
+    '''
+    runprogressbar = tqdm(
+        source_folder_dict.keys(),
+        total=len(source_folder_dict),
+        desc="Run progress",
+        unit="plates",
+        colour='cyan' if platform.system() == 'Windows' else "#0AAFAF",
+        leave=True,
+    )
+    for current_plate in runprogressbar:
+        # render all the channels of the plate
+        if parallelism == 1:
+            render_single_plate_plateview(
+                source_folder_dict[current_plate],
+                current_plate,
+                channel_list,
+                output_path,
+                temp_folder_path,
+                keep_temp_files,
+            )
+        else:
+            render_single_plate_plateview_parallelism(
+                source_folder_dict[current_plate],
+                current_plate,
+                channel_list,
+                output_path,
+                temp_folder_path,
+                parallelism,
+                keep_temp_files,
+            )
+
+    print(os.linesep + os.linesep + "Run completed!")
+    print(str(len(source_folder_dict.keys())), "plate(s) have been processed.", os.linesep)
+
+    return
+
+
+
+
+
+
+
+ +
+ + + \ No newline at end of file diff --git a/docs/index.html b/docs/index.html new file mode 100644 index 0000000..0550517 --- /dev/null +++ b/docs/index.html @@ -0,0 +1,89 @@ + + + + + + +lumos API documentation + + + + + + + + + + + +
+
+
+

Package lumos

+
+
+

Lumos - Back-end module

+
+ +Expand source code + +
'''
+Lumos - Back-end module
+'''
+
+
+
+

Sub-modules

+
+
lumos.generator
+
+

Main functions to generate platemaps with lumos.

+
+
lumos.logger
+
+

Logger functions for lumos.

+
+
lumos.parameters
+
+

Main parameters for lumos operation.

+
+
lumos.picasso
+
+

Main functions to generate cell-painted platemaps with Lumos Picasso.

+
+
lumos.toolbox
+
+

Extra helper functions for lumos.

+
+
+
+
+
+
+
+
+
+
+ +
+ + + \ No newline at end of file diff --git a/docs/logger.html b/docs/logger.html new file mode 100644 index 0000000..363421a --- /dev/null +++ b/docs/logger.html @@ -0,0 +1,390 @@ + + + + + + +lumos.logger API documentation + + + + + + + + + + + +
+
+
+

Module lumos.logger

+
+
+

Logger functions for lumos.

+
+ +Expand source code + +
#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+'''
+Logger functions for lumos.
+'''
+from logging.handlers import RotatingFileHandler
+import logging
+import os
+
+
+# state variable to know if the current lumos session is using parallelism
+_is_in_parallel = True
+
+
+def setup(temp_directory, is_in_parallel):
+    '''
+    Sets-up the logger inside of the working temporary directory. If parallelism is enabled, no logs are stored (they will be printed to the console by default).
+
+            Parameters:
+                    temp_directory (Path): The path to the temporary directory.
+                    parallelism (bool): Whether or not parallel computation is enabled for the current run of the program.
+    '''
+
+    # set the global state variable of the module
+    global _is_in_parallel
+    _is_in_parallel = is_in_parallel
+
+    if not _is_in_parallel:
+
+        # define log format
+        log_formatter = logging.Formatter('%(asctime)s %(levelname)s:\t%(message)s')
+
+        # create logger
+        app_log = logging.getLogger('root')
+
+        # create a rotating log file for regular execution (3 files * 2MB max)
+        my_handler = RotatingFileHandler(
+            temp_directory + "/lumos.log",
+            mode='a',
+            maxBytes=2*1024*1024,
+            backupCount=2,
+            encoding=None,
+            delay=0
+        )
+
+        my_handler.setFormatter(log_formatter)
+        my_handler.setLevel(logging.DEBUG)
+
+        app_log.setLevel(logging.DEBUG)
+        app_log.addHandler(my_handler)
+
+    else:
+        # don't log anything, as it is not compatible with multiprocessing
+        pass
+
+
+def p_print(text, end=os.linesep):
+    '''
+    Parallel print: Handles printing to the console, according to if parallelism is being used or not.
+
+            Parameters:
+                    text (string): The text to be printed.
+                    end (string): What the separating character at the end of the print should be.
+    '''
+    if not _is_in_parallel:
+        print(text, end=end)
+
+
+def debug(text):
+    '''
+    Stores the message as a DEBUG log in the log file (according to if parallelism is being used or not).
+
+            Parameters:
+                    text (string): The message to be stored.
+    '''
+    if not _is_in_parallel:
+        logging.getLogger('root').debug(text)
+
+
+def info(text):
+    '''
+    Stores the message as an INFO log in the log file (according to if parallelism is being used or not).
+
+            Parameters:
+                    text (string): The message to be stored.
+    '''
+    if not _is_in_parallel:
+        logging.getLogger('root').info(text)
+
+
+def warning(text):
+    '''
+    Stores the message as a WARNING log in the log file (according to if parallelism is being used or not).
+
+            Parameters:
+                    text (string): The message to be stored.
+    '''
+    if not _is_in_parallel:
+        logging.getLogger('root').warning(text)
+
+
+def error(text):
+    '''
+    Stores the message as an ERROR log in the log file (according to if parallelism is being used or not).
+
+            Parameters:
+                    text (string): The message to be stored.
+    '''
+    if not _is_in_parallel:
+        logging.getLogger('root').error(text)
+
+
+def critical(text):
+    '''
+    Stores the message as a CRITICAL log in the log file (according to if parallelism is being used or not).
+
+            Parameters:
+                    text (string): The message to be stored.
+    '''
+    if not _is_in_parallel:
+        logging.getLogger('root').critical(text)
+
+
+
+
+
+
+
+

Functions

+
+
+def critical(text) +
+
+

Stores the message as a CRITICAL log in the log file (according to if parallelism is being used or not).

+
    Parameters:
+            text (string): The message to be stored.
+
+
+ +Expand source code + +
def critical(text):
+    '''
+    Stores the message as a CRITICAL log in the log file (according to if parallelism is being used or not).
+
+            Parameters:
+                    text (string): The message to be stored.
+    '''
+    if not _is_in_parallel:
+        logging.getLogger('root').critical(text)
+
+
+
+def debug(text) +
+
+

Stores the message as a DEBUG log in the log file (according to if parallelism is being used or not).

+
    Parameters:
+            text (string): The message to be stored.
+
+
+ +Expand source code + +
def debug(text):
+    '''
+    Stores the message as a DEBUG log in the log file (according to if parallelism is being used or not).
+
+            Parameters:
+                    text (string): The message to be stored.
+    '''
+    if not _is_in_parallel:
+        logging.getLogger('root').debug(text)
+
+
+
+def error(text) +
+
+

Stores the message as an ERROR log in the log file (according to if parallelism is being used or not).

+
    Parameters:
+            text (string): The message to be stored.
+
+
+ +Expand source code + +
def error(text):
+    '''
+    Stores the message as an ERROR log in the log file (according to if parallelism is being used or not).
+
+            Parameters:
+                    text (string): The message to be stored.
+    '''
+    if not _is_in_parallel:
+        logging.getLogger('root').error(text)
+
+
+
+def info(text) +
+
+

Stores the message as an INFO log in the log file (according to if parallelism is being used or not).

+
    Parameters:
+            text (string): The message to be stored.
+
+
+ +Expand source code + +
def info(text):
+    '''
+    Stores the message as an INFO log in the log file (according to if parallelism is being used or not).
+
+            Parameters:
+                    text (string): The message to be stored.
+    '''
+    if not _is_in_parallel:
+        logging.getLogger('root').info(text)
+
+
+
+def p_print(text, end='\n') +
+
+

Parallel print: Handles printing to the console, according to if parallelism is being used or not.

+
    Parameters:
+            text (string): The text to be printed.
+            end (string): What the separating character at the end of the print should be.
+
+
+ +Expand source code + +
def p_print(text, end=os.linesep):
+    '''
+    Parallel print: Handles printing to the console, according to if parallelism is being used or not.
+
+            Parameters:
+                    text (string): The text to be printed.
+                    end (string): What the separating character at the end of the print should be.
+    '''
+    if not _is_in_parallel:
+        print(text, end=end)
+
+
+
+def setup(temp_directory, is_in_parallel) +
+
+

Sets-up the logger inside of the working temporary directory. If parallelism is enabled, no logs are stored (they will be printed to the console by default).

+
    Parameters:
+            temp_directory (Path): The path to the temporary directory.
+            parallelism (bool): Whether or not parallel computation is enabled for the current run of the program.
+
+
+ +Expand source code + +
def setup(temp_directory, is_in_parallel):
+    '''
+    Sets-up the logger inside of the working temporary directory. If parallelism is enabled, no logs are stored (they will be printed to the console by default).
+
+            Parameters:
+                    temp_directory (Path): The path to the temporary directory.
+                    parallelism (bool): Whether or not parallel computation is enabled for the current run of the program.
+    '''
+
+    # set the global state variable of the module
+    global _is_in_parallel
+    _is_in_parallel = is_in_parallel
+
+    if not _is_in_parallel:
+
+        # define log format
+        log_formatter = logging.Formatter('%(asctime)s %(levelname)s:\t%(message)s')
+
+        # create logger
+        app_log = logging.getLogger('root')
+
+        # create a rotating log file for regular execution (3 files * 2MB max)
+        my_handler = RotatingFileHandler(
+            temp_directory + "/lumos.log",
+            mode='a',
+            maxBytes=2*1024*1024,
+            backupCount=2,
+            encoding=None,
+            delay=0
+        )
+
+        my_handler.setFormatter(log_formatter)
+        my_handler.setLevel(logging.DEBUG)
+
+        app_log.setLevel(logging.DEBUG)
+        app_log.addHandler(my_handler)
+
+    else:
+        # don't log anything, as it is not compatible with multiprocessing
+        pass
+
+
+
+def warning(text) +
+
+

Stores the message as a WARNING log in the log file (according to if parallelism is being used or not).

+
    Parameters:
+            text (string): The message to be stored.
+
+
+ +Expand source code + +
def warning(text):
+    '''
+    Stores the message as a WARNING log in the log file (according to if parallelism is being used or not).
+
+            Parameters:
+                    text (string): The message to be stored.
+    '''
+    if not _is_in_parallel:
+        logging.getLogger('root').warning(text)
+
+
+
+
+
+
+
+ +
+ + + \ No newline at end of file diff --git a/docs/parameters.html b/docs/parameters.html new file mode 100644 index 0000000..38fb02a --- /dev/null +++ b/docs/parameters.html @@ -0,0 +1,212 @@ + + + + + + +lumos.parameters API documentation + + + + + + + + + + + +
+
+
+

Module lumos.parameters

+
+
+

Main parameters for lumos operation.

+
+ +Expand source code + +
#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+'''
+Main parameters for lumos operation.
+'''
+
+cellplainting_channels_info = [
+    # Channel number, Channel name, EX wavelength, RGB equivalence
+    ["C01", "C01 DNA Hoechst 33342", 450, [0, 70, 255]],            # ~blue
+    ["C02", "C02 ER Concanavalin A", 510, [0, 255, 0]],             # ~green
+    ["C03", "C03 RNA SYTO 14", 570, [225, 255, 0]],                 # ~yellow
+    ["C04", "C04 AGP Phalloidin and WGA", 630, [255, 79, 0]],       # ~orange
+    ["C05", "C05 MITO MitoTracker Deep Red", 660, [255, 0, 0]],     # ~red
+    ["Z01C06", "C06 Brigtfield depth1", None, None],
+    ["Z02C06", "C06 Brigtfield depth2", None, None],
+    ["Z03C06", "C06 Brigtfield depth3", None, None],
+]
+'''
+Matrix of information for each of the channels.
+    Columns: [Channel number, Channel name, EX wavelength, RGB equivalence]
+'''
+
+cellplainting_channels_dict = {
+    "C01": "DNA Hoechst 33342",
+    "C02": "ER Concanavalin A",
+    "C03": "RNA SYTO 14",
+    "C04": "AGP Phalloidin and WGA",
+    "C05": "MITO MitoTracker Deep Red",
+    "Z01C06": "Brightfield depth1",
+    "Z02C06": "Brightfield depth2",
+    "Z03C06": "Brightfield depth3",
+}
+'''
+Dictionary of the channel names and their respective labels.
+'''
+
+# what are the default channels to render for a plate/run
+default_channels_to_render = [
+    "C01",
+    "C02",
+    "C03",
+    "C04",
+    "C05",
+]
+'''
+List of the default channels to render in a run or single plate rendering.
+'''
+
+# intensity normalizing coefficient factors per channel
+channel_coefficients = {
+    "C01": 16,
+    "C02": 8,
+    "C03": 8,
+    "C04": 8,
+    "C05": 8,
+    "Z01C06": 8,
+    "Z02C06": 8,
+    "Z03C06": 8,
+}
+'''
+Intensity multiplier coefficients for each of the channels (those are arbitrary and used to make interest points easier to see).
+'''
+
+clipping_threshold_min_value = 1
+clipping_threshold_max_value = 12000
+normalize_alpha = 0
+normalize_beta = 65535
+rescale_ratio = 0.1
+
+placeholder_background_intensity = 64
+placeholder_markers_intensity = 0
+
+
+#  --------  PARAMETERS ONLY FOR CELL-PAINTING (PICASSO)  --------
+
+rescale_ratio_picasso_wells = 1
+rescale_ratio_picasso_plate = 0.25
+
+# list of merge rendering styles for picasso
+fingerprint_style_dict = {
+    'accurate': [[], [], []],
+    'random': [[], [], []],
+    'blueish': [[6, 5, 6, 6, 6], [2, 3, 4, 1, 0], [0, 2, 1]],
+    'blueish2': [[4, 6, 5, 5, 7], [3, 2, 0, 4, 1], [0, 1, 2]],
+    'blueredgreen': [[3, 8, 4, 4, 8], [0, 3, 4, 2, 1], [2, 0, 1]],
+    'blueredgreen2': [[3, 4, 4, 5, 6], [2, 3, 4, 1, 0], [2, 1, 0]],
+    'blueredgreen3': [[8, 4, 6, 5, 8], [1, 3, 4, 2, 0], [0, 1, 2]],
+    'reddish': [[7, 7, 4, 4, 1], [2, 1, 3, 4, 0], [1, 0, 2]],
+    'reddish2': [[7, 3, 6, 8, 5], [1, 2, 3, 0, 4], [1, 0, 2]],
+    'purple': [[2, 6, 6, 7, 2], [3, 1, 2, 4, 0], [0, 1, 2]],
+    'purple2': [[1, 7, 8, 6, 8], [2, 4, 0, 3, 1], [0, 1, 2]],
+    'chthulu': [[3, 2, 3, 5, 7], [0, 3, 2, 1, 4], [1, 0, 2]],
+    'meduse': [[8, 8, 3, 7, 8], [0, 3, 4, 1, 2], [2, 0, 1]],
+    'alien': [[3, 6, 4, 3, 3], [1, 3, 2, 4, 0], [1, 0, 2]],
+}
+'''
+Dictionary of the styles that can be used for "merge cell painting", and their associated coefficients.
+'''
+
+accurate_style_parameters = {
+    'intensity': [11, 9, 2, 4, 12],
+    'contrast': [0, 0, 0.5, 1, 1.85],
+}
+'''
+Dictionary of the coefficients used for "accurate" cell painting (using an approximation of the actual colors of the channels emitted wavelengths)
+'''
+
+# Other styles not integrated
+#     'darkgreenblue': [[3,6,2,2,8],[2,1,3,0,4],[2,0,1]],
+#     'fingerprint1': [[6,2,3,3,2],[3,1,0,4,2],[0,1,2]],
+#     'fingerprint3': [[7,7,6,4,3],[0,2,1,3,4],[1,0,2]],
+#     'fingerprint2': [[2,6,6,1,5],[4,3,1,2,0],[2,0,1]],
+#     'fingerprint6': [[8,7,4,4,7],[1,3,2,4,0],[0,1,2]],
+#     'fingerprint8': [[2,1,4,6,5],[0,3,2,4,1],[0,2,1]],
+
+
+
+
+
+

Global variables

+
+
var accurate_style_parameters
+
+

Dictionary of the coefficients used for "accurate" cell painting (using an approximation of the actual colors of the channels emitted wavelengths)

+
+
var cellplainting_channels_dict
+
+

Dictionary of the channel names and their respective labels.

+
+
var cellplainting_channels_info
+
+

Matrix of information for each of the channels. +Columns: [Channel number, Channel name, EX wavelength, RGB equivalence]

+
+
var channel_coefficients
+
+

Intensity multiplier coefficients for each of the channels (those are arbitrary and used to make interest points easier to see).

+
+
var default_channels_to_render
+
+

List of the default channels to render in a run or single plate rendering.

+
+
var fingerprint_style_dict
+
+

Dictionary of the styles that can be used for "merge cell painting", and their associated coefficients.

+
+
+
+
+
+
+
+
+ +
+ + + \ No newline at end of file diff --git a/docs/picasso.html b/docs/picasso.html new file mode 100644 index 0000000..dd41cb5 --- /dev/null +++ b/docs/picasso.html @@ -0,0 +1,1338 @@ + + + + + + +lumos.picasso API documentation + + + + + + + + + + + +
+
+
+

Module lumos.picasso

+
+
+

Main functions to generate cell-painted platemaps with Lumos Picasso.

+
+ +Expand source code + +
#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+'''
+Main functions to generate cell-painted platemaps with Lumos Picasso.
+'''
+
+import math
+import os
+from pathlib import Path
+import pandas as pd
+from shutil import copyfile
+import shutil
+from . import toolbox
+from . import parameters
+import cv2
+from tqdm import tqdm
+from . import logger
+import numpy as np
+import random
+
+
+def colorizer(
+    img_channels_fullpath,
+    rescale_ratio,
+    style,
+    max_multiply_coef=1,
+    display_fingerprint=False,
+):
+    '''
+    Merges input images from different channels into one RGB image.
+
+            Parameters:
+                    img_channels_fullpath (Path list): The list of paths to the channels' images (in proper order [C01,C02,C03,C04,C05]).
+                    rescale_ratio (float): The ratio used to rescale the image before generation.
+                    style (string): The name of the style being used to generate the colorized image.
+                    max_multiply_coef (int): Max multiplication factor in case of random coefficient generation.
+                    display_fingerprint (bool): Whether the coefficients used for generation should be printed on the output image.
+
+            Returns:
+                    8-bit cv2 image
+    '''
+
+    # load images from path list + resize + convert to 8bit
+    np_image_channels_array = []
+    for current_image in img_channels_fullpath:
+        # load image
+        img16 = cv2.imread(str(current_image), -1)
+        try:
+            assert(not img16.shape == (0, 0))
+        except:
+            # create blank file
+            img16 = np.full(shape=(1000, 1000, 1),
+                            fill_value=0, dtype=np.uint16)
+            logger.warning("Missing or corrupted image " + str(current_image))
+
+        # resize image
+        img16 = cv2.resize(
+            src=img16,
+            dsize=None,
+            fx=rescale_ratio,
+            fy=rescale_ratio,
+            interpolation=cv2.INTER_CUBIC,
+        )
+
+        # convert image to 8bit
+        img8 = (img16 / 256).astype("uint8")
+        np_image_channels_array.append(img8)
+
+    # Perform merging, according to the style
+    if style == 'accurate':
+        # initialize RGB channels
+        red_channel = np.zeros(np_image_channels_array[0].shape)
+        green_channel = np.zeros(np_image_channels_array[0].shape)
+        blue_channel = np.zeros(np_image_channels_array[0].shape)
+
+        # # compute the mean of each layer
+        # means=[]
+        # for idx in range(5):
+        #     means.append(np.mean(np_image_channels_array[idx]))
+
+        # # contrast image at the mean of each layer (naive approche)
+        # for idx in range(5):
+        #     vLambda = np.vectorize(lambda x : ((x**parameters.accurate_style_parameters['contrast_coeffs'][idx])/means[idx]) if parameters.accurate_style_parameters['contrast_coeffs'][idx] != 0 else x)
+        #     np_image_channels_array[idx] = vLambda(np_image_channels_array[idx])
+
+        # # perform thresholding at the mean of each layer
+        # for idx in range(5):
+        #     # thresholder = lambda x : x if x > (means[idx] * parameters.accurate_style_parameters['threshold_coeffs'][idx])  else 0
+        #     # vThreshold = np.vectorize(thresholder)
+        #     # np_image_channels_array[idx] = vThreshold(np_image_channels_array[idx])
+
+        # get the current style's contrast coefficients
+        contrast_coef = parameters.accurate_style_parameters['contrast']
+        # add contrast to each layer according to coefficients
+        for idx in range(5):
+            contrast = contrast_coef[idx]
+            f = float(131 * (contrast + 127)) / (127 * (131 - contrast))
+            alpha_c = f
+            gamma_c = 127*(1-f)
+            np_image_channels_array[idx] = cv2.addWeighted(
+                np_image_channels_array[idx], alpha_c, np_image_channels_array[idx], 0, gamma_c)
+
+        # get the current style's intensity coefficients
+        intensity_coef = parameters.accurate_style_parameters['intensity']
+        # multiply the intensity of the channels using input coefs
+        np_image_array_adjusted = []
+        for idx in range(5):  # TODO: adapt for less than 5 selected channels
+            np_image_array_adjusted.append(
+                np_image_channels_array[idx] * intensity_coef[idx])
+        np_image_channels_array = np_image_array_adjusted
+
+        # combine the images according to their RGB coefficients
+        for idx in range(5):
+            red_channel = red_channel + \
+                (np_image_channels_array[idx] / 255 *
+                 parameters.cellplainting_channels_info[idx][3][0])
+            green_channel = green_channel + \
+                (np_image_channels_array[idx] / 255 *
+                 parameters.cellplainting_channels_info[idx][3][1])
+            blue_channel = blue_channel + \
+                (np_image_channels_array[idx] / 255 *
+                 parameters.cellplainting_channels_info[idx][3][2])
+
+        # merge the Blue, Green and Red channels to form the final image
+        merged_img = cv2.merge(
+            (blue_channel, green_channel, red_channel)
+        )
+
+    else:
+
+        # get the current style's intensity coefficients
+        intensity_coef = parameters.fingerprint_style_dict[style][0]
+
+        # get other parameters
+        channel_order = parameters.fingerprint_style_dict[style][1]
+        target_rgb = parameters.fingerprint_style_dict[style][2]
+
+        # randomly initiate coefficients if they are missing
+        if len(intensity_coef) == 0 and len(channel_order) == 0 and len(target_rgb) == 0:
+            # parameters for each channel
+            intensity_coef = [random.randint(
+                1, max_multiply_coef) for x in range(5)]
+            channel_order = [0, 1, 2, 3, 4]
+            random.shuffle(channel_order)
+            target_rgb = [0, 1, 2]
+            random.shuffle(target_rgb)
+
+        # multiply the intensity of the channels using input coefs
+        np_image_array_adjusted = []
+        # TODO: adapt for less than 5 selected channels?
+        for index, current_coef_mult in enumerate(intensity_coef):
+            np_image_array_adjusted.append(
+                np_image_channels_array[index] * current_coef_mult)
+        np_image_channels_array = np_image_array_adjusted
+
+        # merge 2 extra channels each on 1 rgb channel
+        np_image_channels_array[target_rgb[0]] = (
+            np_image_channels_array[target_rgb[0]] +
+            np_image_channels_array[channel_order[3]]
+        )
+        np_image_channels_array[target_rgb[1]] = (
+            np_image_channels_array[target_rgb[1]] +
+            np_image_channels_array[channel_order[4]]
+        )
+
+        merged_img = cv2.merge(
+            (
+                np_image_channels_array[channel_order[0]],
+                np_image_channels_array[channel_order[1]],
+                np_image_channels_array[channel_order[2]],
+            )
+        )
+
+    # add fingerprint id on image
+    if display_fingerprint:
+        text = str(intensity_coef) + str(channel_order) + str(
+            target_rgb) if style != 'accurate' else str(parameters.accurate_style_parameters)
+        font = cv2.FONT_HERSHEY_SIMPLEX
+        cv2.putText(
+            merged_img,
+            text,
+            (math.ceil(10*rescale_ratio), math.ceil(990*rescale_ratio)),
+            font,
+            0.8*rescale_ratio,
+            (192, 192, 192),
+            math.ceil(2*rescale_ratio),
+            cv2.INTER_AREA,
+        )
+
+    return merged_img
+
+
+def generate_multiplexed_well_images(
+    data_df, temp_folder, style, display_well_details, scope
+):
+    '''
+    Generates a colorized image from all 5 channels of a well, for all wells, and saves it in the temporary directory.
+
+            Parameters:
+                    data_df (Pandas DataFrame): Dataframe containing the paths to each channel, of each site, of each well.
+                    temp_folder_path (Path): The path to the folder where temporary data can be stored.
+                    style (string): The name of the style being used to generate the colorized image.
+                    style (string): The name of rendering style.
+                    display_well_details (bool): Whether or not the name of the well should be printed on its generated image.
+                    scope (string): 'plate' or 'wells' (this will have an impact on the resizing of the well/site images).
+
+            Returns:
+                    True (in case of success)
+    '''
+
+    if scope == 'wells':
+        rescale_ratio = parameters.rescale_ratio_picasso_wells
+    if scope == 'plate':
+        rescale_ratio = parameters.rescale_ratio_picasso_plate
+
+    # get the well list
+    well_list = list(set(data_df["well"]))
+    well_list.sort()
+
+    # multiplex all well/site channels into 1 well/site 8bit color image
+    wellprogressbar = tqdm(list(well_list), unit="wells", leave=False)
+    for current_well in wellprogressbar:
+        current_well_sites_multiplexed_image_list = []
+        for current_site in range(1, 7):
+
+            # get the image path list for the channels of the site, in the correct order
+            current_sites_df = data_df[
+                ((data_df["well"] == current_well) &
+                 (data_df["site"] == current_site))
+            ]
+            current_sites_df_ordered = current_sites_df.sort_values(
+                by="channel", ascending=True
+            )
+
+            channel_images_path = current_sites_df_ordered["fullpath"].to_list()
+
+            # proceed to the generation using shaker 4 function with a first predefined fingerprint
+            multiplexed_image = colorizer(
+                img_channels_fullpath=channel_images_path,
+                rescale_ratio=rescale_ratio,
+                max_multiply_coef=8,
+                style=style,
+            )
+
+            # collect image in memory
+            current_well_sites_multiplexed_image_list.append(multiplexed_image)
+
+        # save well image
+        sites_row1 = cv2.hconcat(
+            [
+                current_well_sites_multiplexed_image_list[0],
+                current_well_sites_multiplexed_image_list[1],
+                current_well_sites_multiplexed_image_list[2],
+            ]
+        )
+        sites_row2 = cv2.hconcat(
+            [
+                current_well_sites_multiplexed_image_list[3],
+                current_well_sites_multiplexed_image_list[4],
+                current_well_sites_multiplexed_image_list[5],
+            ]
+        )
+        all_sites_image = cv2.vconcat([sites_row1, sites_row2])
+
+        # add fingerprint id on image
+        if display_well_details:
+            text = str(current_well)
+            font = cv2.FONT_HERSHEY_SIMPLEX
+            cv2.putText(
+                img=all_sites_image,
+                text=text,
+                org=(math.ceil(80*rescale_ratio), math.ceil(80*rescale_ratio)),
+                fontFace=font,
+                fontScale=2.2*rescale_ratio,
+                thickness=math.ceil(3*rescale_ratio),
+                color=(192, 192, 192),
+                lineType=cv2.INTER_AREA,
+            )
+        if scope == 'plate':
+            # add well marks on borders
+            image_shape = all_sites_image.shape
+            cv2.rectangle(
+                all_sites_image,
+                (0, 0),
+                (image_shape[1], image_shape[0]),
+                (192, 192, 192),
+                math.ceil(8*rescale_ratio),
+            )
+
+        cv2.imwrite(
+            temp_folder + "/wells/well-" + str(current_well) + ".png",
+            all_sites_image,
+        )
+
+    return
+
+
+def concatenate_well_images(well_list, temp_folder_path):
+    '''
+    Loads all temporary well images from the temporary directory and concatenates them into one image of the whole plate.
+
+            Parameters:
+                    well_list (string list): A list of all the well IDs (e.g. ['A01', 'A02', 'A03', ...]).
+                    temp_folder_path (Path): The path to the folder where temporary data can be stored.
+
+            Returns:
+                    8-bit cv2 image: The concatenated image of all the wells
+    '''
+
+    # load all well images and store images in memory into a list
+    print("Load well images in memory..")
+    logger.info("Load well images in memory..")
+
+    image_well_data = []
+    for current_well in list(well_list):
+        well_image = toolbox.load_well_image(
+            current_well,
+            temp_folder_path + "/wells",
+        )
+        image_well_data.append(well_image)
+
+    # concatenate all the well images into horizontal stripes (1 per row)
+    logger.info("Concatenate images into a plate..")
+
+    image_row_data = []
+    for current_plate_row in range(1, 17):
+
+        # concatenate horizontally and vertically
+        well_start_id = ((current_plate_row - 1) * 24) + 0
+        well_end_id = current_plate_row * 24
+        sites_row = cv2.hconcat(image_well_data[well_start_id:well_end_id])
+        image_row_data.append(sites_row)
+
+    # concatenate all the stripes into 1 image
+    plate_image = cv2.vconcat(image_row_data)
+    return plate_image
+
+
+def get_images_full_path(channel_string_ids, plate_input_path):
+    '''
+    Finds all the paths to all the channels' images from the input folder
+
+            Parameters:
+                    channel_string_ids (string list): A list of all the channels IDs to be loaded.
+                    plate_input_path (Path): The path to the folder where the input images are stored.
+
+            Returns:
+                    Path list: A list of all the paths to all images of each channels
+    '''
+
+    # get the files from the plate folder, for the targeted channel
+    images_full_path_list = []
+    for current_channel in channel_string_ids:
+        current_channel_images_full_path_list = list(
+            Path(plate_input_path).glob("*" + current_channel + ".tif")
+        )
+        images_full_path_list = images_full_path_list + \
+            current_channel_images_full_path_list
+
+    # check that we get expected images for a 384 well image
+    try:
+        assert len(images_full_path_list) == 2304 * 5
+    except AssertionError:
+        print(
+            "The plate does not have the exact image count: expected " +
+            str(2304 * 5) + ", got "
+            + str(len(images_full_path_list)),
+        )
+
+    return images_full_path_list
+
+
+def build_robustized_plate_dataframe(images_full_path_list):
+    '''
+    Scans the input list of Paths to map it to the expected plate structure.
+    Missing images or wells must be taken into account in the final render.
+
+            Parameters:
+                    images_full_path_list (Path list): A list of all the paths to all the images to be included in the render.
+
+            Returns:
+                    Pandas DataFrame:
+                            A database of all the image paths to each channel, of each site, of each well.
+                            Its columns are: ["well", "site", "channel", "filename", "fullpath"].
+    '''
+
+    # get the filenames list
+    images_full_path_list.sort()
+    images_filename_list = [str(x.name) for x in images_full_path_list]
+
+    # get the well list
+    image_well_list = [x.split("_")[1].split("_T")[0]
+                       for x in images_filename_list]
+
+    # get the siteid list (sitesid from 1 to 6)
+    image_site_list = [
+        x.split("_T0001F")[1].split("L")[0] for x in images_filename_list
+    ]
+    image_site_list_int = [int(x) for x in image_site_list]
+
+    # get the channel id list (channel id from 1 to 5)
+    image_channel_list = [x.split(".ti")[0][-2:] for x in images_filename_list]
+    image_channel_list_int = [int(x) for x in image_channel_list]
+
+    # zip all in a data structure
+    image_data_zip = zip(
+        image_well_list,
+        image_site_list_int,
+        image_channel_list_int,
+        images_filename_list,
+        images_full_path_list,
+    )
+
+    # convert the zip into dataframe
+    data_df = pd.DataFrame(
+        list(image_data_zip),
+        columns=["well", "site", "channel", "filename", "fullpath"],
+    )
+
+    # get the theoretical well list for 384 well plate
+    well_theoretical_list = [
+        l + str(r).zfill(2) for l in "ABCDEFGHIJKLMNOP" for r in range(1, 25)
+    ]
+    well_channel_theoretical_list = [
+        [x, r, c] for x in well_theoretical_list for r in range(1, 7) for c in range(1, 6)
+    ]
+
+    # create the theoretical well dataframe
+    theoretical_data_df = pd.DataFrame(
+        well_channel_theoretical_list, columns=["well", "site", "channel"]
+    )
+
+    # join the real wells with the theoric ones
+    theoretical_data_df_joined = theoretical_data_df.merge(
+        data_df,
+        left_on=["well", "site", "channel"],
+        right_on=["well", "site", "channel"],
+        how="left",
+    )
+
+    # log if there is a delta between theory and actual plate wells
+    delta = set(well_theoretical_list) - set(image_well_list)
+    logger.info("Well Delta " + str(delta))
+
+    return theoretical_data_df_joined
+
+
+def copy_well_images_to_output_folder(temp_folder, output_path, well_list, plate_name, style):
+    '''
+    Copies all temporary well images into the output folder.
+    Used for when the scope of the operation is 'well' and we want only the well images to be outputed.
+
+            Parameters:
+                    temp_folder (Path): The path to the temporary working directory where the well images are currently stored in.
+                    output_path (Path): The path to the folder where the images should be copied to.
+                    well_list (string list): A list of all the well IDs (e.g. ['A01', 'A02', 'A03', ...]).
+                    plate_name (string): The name of the current plate (used to generate the output files' names).
+                    style (string): The name of the style used for rendering (used to generate the output files' names).
+
+            Returns:
+                    8 bit cv2 image
+    '''
+
+    print("Putting well images into output folder..")
+
+    for current_well in list(well_list):
+        copyfile(
+            temp_folder + "/wells/well-"+current_well+".png",
+            output_path+'/'+plate_name+"-"+current_well+"-"+style+".png"
+        )
+
+    return
+
+
+def picasso_generate_plate_image(
+    source_path,
+    plate_name,
+    output_path,
+    temp_folder_path,
+    style,
+    scope,
+    display_well_details,
+):
+    '''
+    Generates cell-painted colorized images of individual wells or of a whole plate.
+
+            Parameters:
+                    source_path (Path): The folder where the input images of the plate are stored.
+                    plate_name (string): The name of the plate being rendered.
+                    output_path (Path): The path to the folder where the output images should be stored.
+                    temp_folder_path (Path): The path to the folder where temporary data can be stored.
+                    style (string): The name of the rendering style.
+                    scope (string):
+                            Either 'wells' or 'plate'. Defines if we should generate individual well images,
+                            or concatenate them into a single plate image.
+                    display_well_details (bool): Whether or not the name of the well should be written on the generated images.
+
+            Returns:
+                    8 bit cv2 image(s):
+                            If the scope is 'wells', then all colorized well images are outputed to the output folder.
+                            If the scope is 'wells', then the well images are concatenated into one image of the whole
+                            plate before being outputed to the output folder.
+    '''
+
+    # define a temp folder for the run
+    temp_folder_path = temp_folder_path + "/tmpgen-" + plate_name + "picasso"
+
+    # remove temp dir if existing
+    shutil.rmtree(temp_folder_path, ignore_errors=True)
+
+    # create the temporary directory structure to work on wells
+    try:
+        os.mkdir(temp_folder_path)
+    except FileExistsError:
+        pass
+    # also create a subfolder to store well images
+    try:
+        os.mkdir(temp_folder_path + "/wells")
+    except FileExistsError:
+        pass
+
+    # read the plate input path
+    plate_input_path = Path(source_path)
+
+    # get the list of all paths for each channel image
+    images_full_path_list = get_images_full_path(
+        # TODO: adapt for less than 5 selected channels?
+        channel_string_ids=["C01", "C02", "C03", "C04", "C05"],
+        plate_input_path=plate_input_path,
+    )
+
+    # build a database of the theorical plate
+    # TODO: adapt for less than 5 selected channels?
+    data_df = build_robustized_plate_dataframe(images_full_path_list)
+
+    # get the well list
+    well_list = list(set(data_df["well"]))
+    well_list.sort()
+
+    # generate images inside the temp folder
+    generate_multiplexed_well_images(
+        data_df=data_df,
+        temp_folder=temp_folder_path,
+        style=style,
+        display_well_details=display_well_details,
+        scope=scope,
+    )
+
+    if scope == 'plate':
+        # concatenate well images into a plate image
+        plate_image = concatenate_well_images(well_list, temp_folder_path)
+
+        # save image
+        plate_image_path = (
+            output_path + "/" + plate_name + "-" +
+            "picasso" + "-" + str(style) + ".jpg"
+        )
+        cv2.imwrite(plate_image_path, plate_image)
+
+        print(" -> Generated image of size:", plate_image.shape)
+        print(" -> Saved as ", plate_image_path)
+
+    if scope == 'wells':
+        # copy well files in output folder
+        copy_well_images_to_output_folder(
+            temp_folder_path, output_path, well_list, plate_name, style)
+
+        print(" -> Saved well images in ", output_path)
+
+    # purge temp files
+    logger.info("Purge temporary folder")
+    shutil.rmtree(temp_folder_path, ignore_errors=True)
+
+    return
+
+
+
+
+
+
+
+

Functions

+
+
+def build_robustized_plate_dataframe(images_full_path_list) +
+
+

Scans the input list of Paths to map it to the expected plate structure. +Missing images or wells must be taken into account in the final render.

+
    Parameters:
+            images_full_path_list (Path list): A list of all the paths to all the images to be included in the render.
+
+    Returns:
+            Pandas DataFrame:
+                    A database of all the image paths to each channel, of each site, of each well.
+                    Its columns are: ["well", "site", "channel", "filename", "fullpath"].
+
+
+ +Expand source code + +
def build_robustized_plate_dataframe(images_full_path_list):
+    '''
+    Scans the input list of Paths to map it to the expected plate structure.
+    Missing images or wells must be taken into account in the final render.
+
+            Parameters:
+                    images_full_path_list (Path list): A list of all the paths to all the images to be included in the render.
+
+            Returns:
+                    Pandas DataFrame:
+                            A database of all the image paths to each channel, of each site, of each well.
+                            Its columns are: ["well", "site", "channel", "filename", "fullpath"].
+    '''
+
+    # get the filenames list
+    images_full_path_list.sort()
+    images_filename_list = [str(x.name) for x in images_full_path_list]
+
+    # get the well list
+    image_well_list = [x.split("_")[1].split("_T")[0]
+                       for x in images_filename_list]
+
+    # get the siteid list (sitesid from 1 to 6)
+    image_site_list = [
+        x.split("_T0001F")[1].split("L")[0] for x in images_filename_list
+    ]
+    image_site_list_int = [int(x) for x in image_site_list]
+
+    # get the channel id list (channel id from 1 to 5)
+    image_channel_list = [x.split(".ti")[0][-2:] for x in images_filename_list]
+    image_channel_list_int = [int(x) for x in image_channel_list]
+
+    # zip all in a data structure
+    image_data_zip = zip(
+        image_well_list,
+        image_site_list_int,
+        image_channel_list_int,
+        images_filename_list,
+        images_full_path_list,
+    )
+
+    # convert the zip into dataframe
+    data_df = pd.DataFrame(
+        list(image_data_zip),
+        columns=["well", "site", "channel", "filename", "fullpath"],
+    )
+
+    # get the theoretical well list for 384 well plate
+    well_theoretical_list = [
+        l + str(r).zfill(2) for l in "ABCDEFGHIJKLMNOP" for r in range(1, 25)
+    ]
+    well_channel_theoretical_list = [
+        [x, r, c] for x in well_theoretical_list for r in range(1, 7) for c in range(1, 6)
+    ]
+
+    # create the theoretical well dataframe
+    theoretical_data_df = pd.DataFrame(
+        well_channel_theoretical_list, columns=["well", "site", "channel"]
+    )
+
+    # join the real wells with the theoric ones
+    theoretical_data_df_joined = theoretical_data_df.merge(
+        data_df,
+        left_on=["well", "site", "channel"],
+        right_on=["well", "site", "channel"],
+        how="left",
+    )
+
+    # log if there is a delta between theory and actual plate wells
+    delta = set(well_theoretical_list) - set(image_well_list)
+    logger.info("Well Delta " + str(delta))
+
+    return theoretical_data_df_joined
+
+
+
+def colorizer(img_channels_fullpath, rescale_ratio, style, max_multiply_coef=1, display_fingerprint=False) +
+
+

Merges input images from different channels into one RGB image.

+
    Parameters:
+            img_channels_fullpath (Path list): The list of paths to the channels' images (in proper order [C01,C02,C03,C04,C05]).
+            rescale_ratio (float): The ratio used to rescale the image before generation.
+            style (string): The name of the style being used to generate the colorized image.
+            max_multiply_coef (int): Max multiplication factor in case of random coefficient generation.
+            display_fingerprint (bool): Whether the coefficients used for generation should be printed on the output image.
+
+    Returns:
+            8-bit cv2 image
+
+
+ +Expand source code + +
def colorizer(
+    img_channels_fullpath,
+    rescale_ratio,
+    style,
+    max_multiply_coef=1,
+    display_fingerprint=False,
+):
+    '''
+    Merges input images from different channels into one RGB image.
+
+            Parameters:
+                    img_channels_fullpath (Path list): The list of paths to the channels' images (in proper order [C01,C02,C03,C04,C05]).
+                    rescale_ratio (float): The ratio used to rescale the image before generation.
+                    style (string): The name of the style being used to generate the colorized image.
+                    max_multiply_coef (int): Max multiplication factor in case of random coefficient generation.
+                    display_fingerprint (bool): Whether the coefficients used for generation should be printed on the output image.
+
+            Returns:
+                    8-bit cv2 image
+    '''
+
+    # load images from path list + resize + convert to 8bit
+    np_image_channels_array = []
+    for current_image in img_channels_fullpath:
+        # load image
+        img16 = cv2.imread(str(current_image), -1)
+        try:
+            assert(not img16.shape == (0, 0))
+        except:
+            # create blank file
+            img16 = np.full(shape=(1000, 1000, 1),
+                            fill_value=0, dtype=np.uint16)
+            logger.warning("Missing or corrupted image " + str(current_image))
+
+        # resize image
+        img16 = cv2.resize(
+            src=img16,
+            dsize=None,
+            fx=rescale_ratio,
+            fy=rescale_ratio,
+            interpolation=cv2.INTER_CUBIC,
+        )
+
+        # convert image to 8bit
+        img8 = (img16 / 256).astype("uint8")
+        np_image_channels_array.append(img8)
+
+    # Perform merging, according to the style
+    if style == 'accurate':
+        # initialize RGB channels
+        red_channel = np.zeros(np_image_channels_array[0].shape)
+        green_channel = np.zeros(np_image_channels_array[0].shape)
+        blue_channel = np.zeros(np_image_channels_array[0].shape)
+
+        # # compute the mean of each layer
+        # means=[]
+        # for idx in range(5):
+        #     means.append(np.mean(np_image_channels_array[idx]))
+
+        # # contrast image at the mean of each layer (naive approche)
+        # for idx in range(5):
+        #     vLambda = np.vectorize(lambda x : ((x**parameters.accurate_style_parameters['contrast_coeffs'][idx])/means[idx]) if parameters.accurate_style_parameters['contrast_coeffs'][idx] != 0 else x)
+        #     np_image_channels_array[idx] = vLambda(np_image_channels_array[idx])
+
+        # # perform thresholding at the mean of each layer
+        # for idx in range(5):
+        #     # thresholder = lambda x : x if x > (means[idx] * parameters.accurate_style_parameters['threshold_coeffs'][idx])  else 0
+        #     # vThreshold = np.vectorize(thresholder)
+        #     # np_image_channels_array[idx] = vThreshold(np_image_channels_array[idx])
+
+        # get the current style's contrast coefficients
+        contrast_coef = parameters.accurate_style_parameters['contrast']
+        # add contrast to each layer according to coefficients
+        for idx in range(5):
+            contrast = contrast_coef[idx]
+            f = float(131 * (contrast + 127)) / (127 * (131 - contrast))
+            alpha_c = f
+            gamma_c = 127*(1-f)
+            np_image_channels_array[idx] = cv2.addWeighted(
+                np_image_channels_array[idx], alpha_c, np_image_channels_array[idx], 0, gamma_c)
+
+        # get the current style's intensity coefficients
+        intensity_coef = parameters.accurate_style_parameters['intensity']
+        # multiply the intensity of the channels using input coefs
+        np_image_array_adjusted = []
+        for idx in range(5):  # TODO: adapt for less than 5 selected channels
+            np_image_array_adjusted.append(
+                np_image_channels_array[idx] * intensity_coef[idx])
+        np_image_channels_array = np_image_array_adjusted
+
+        # combine the images according to their RGB coefficients
+        for idx in range(5):
+            red_channel = red_channel + \
+                (np_image_channels_array[idx] / 255 *
+                 parameters.cellplainting_channels_info[idx][3][0])
+            green_channel = green_channel + \
+                (np_image_channels_array[idx] / 255 *
+                 parameters.cellplainting_channels_info[idx][3][1])
+            blue_channel = blue_channel + \
+                (np_image_channels_array[idx] / 255 *
+                 parameters.cellplainting_channels_info[idx][3][2])
+
+        # merge the Blue, Green and Red channels to form the final image
+        merged_img = cv2.merge(
+            (blue_channel, green_channel, red_channel)
+        )
+
+    else:
+
+        # get the current style's intensity coefficients
+        intensity_coef = parameters.fingerprint_style_dict[style][0]
+
+        # get other parameters
+        channel_order = parameters.fingerprint_style_dict[style][1]
+        target_rgb = parameters.fingerprint_style_dict[style][2]
+
+        # randomly initiate coefficients if they are missing
+        if len(intensity_coef) == 0 and len(channel_order) == 0 and len(target_rgb) == 0:
+            # parameters for each channel
+            intensity_coef = [random.randint(
+                1, max_multiply_coef) for x in range(5)]
+            channel_order = [0, 1, 2, 3, 4]
+            random.shuffle(channel_order)
+            target_rgb = [0, 1, 2]
+            random.shuffle(target_rgb)
+
+        # multiply the intensity of the channels using input coefs
+        np_image_array_adjusted = []
+        # TODO: adapt for less than 5 selected channels?
+        for index, current_coef_mult in enumerate(intensity_coef):
+            np_image_array_adjusted.append(
+                np_image_channels_array[index] * current_coef_mult)
+        np_image_channels_array = np_image_array_adjusted
+
+        # merge 2 extra channels each on 1 rgb channel
+        np_image_channels_array[target_rgb[0]] = (
+            np_image_channels_array[target_rgb[0]] +
+            np_image_channels_array[channel_order[3]]
+        )
+        np_image_channels_array[target_rgb[1]] = (
+            np_image_channels_array[target_rgb[1]] +
+            np_image_channels_array[channel_order[4]]
+        )
+
+        merged_img = cv2.merge(
+            (
+                np_image_channels_array[channel_order[0]],
+                np_image_channels_array[channel_order[1]],
+                np_image_channels_array[channel_order[2]],
+            )
+        )
+
+    # add fingerprint id on image
+    if display_fingerprint:
+        text = str(intensity_coef) + str(channel_order) + str(
+            target_rgb) if style != 'accurate' else str(parameters.accurate_style_parameters)
+        font = cv2.FONT_HERSHEY_SIMPLEX
+        cv2.putText(
+            merged_img,
+            text,
+            (math.ceil(10*rescale_ratio), math.ceil(990*rescale_ratio)),
+            font,
+            0.8*rescale_ratio,
+            (192, 192, 192),
+            math.ceil(2*rescale_ratio),
+            cv2.INTER_AREA,
+        )
+
+    return merged_img
+
+
+
+def concatenate_well_images(well_list, temp_folder_path) +
+
+

Loads all temporary well images from the temporary directory and concatenates them into one image of the whole plate.

+
    Parameters:
+            well_list (string list): A list of all the well IDs (e.g. ['A01', 'A02', 'A03', ...]).
+            temp_folder_path (Path): The path to the folder where temporary data can be stored.
+
+    Returns:
+            8-bit cv2 image: The concatenated image of all the wells
+
+
+ +Expand source code + +
def concatenate_well_images(well_list, temp_folder_path):
+    '''
+    Loads all temporary well images from the temporary directory and concatenates them into one image of the whole plate.
+
+            Parameters:
+                    well_list (string list): A list of all the well IDs (e.g. ['A01', 'A02', 'A03', ...]).
+                    temp_folder_path (Path): The path to the folder where temporary data can be stored.
+
+            Returns:
+                    8-bit cv2 image: The concatenated image of all the wells
+    '''
+
+    # load all well images and store images in memory into a list
+    print("Load well images in memory..")
+    logger.info("Load well images in memory..")
+
+    image_well_data = []
+    for current_well in list(well_list):
+        well_image = toolbox.load_well_image(
+            current_well,
+            temp_folder_path + "/wells",
+        )
+        image_well_data.append(well_image)
+
+    # concatenate all the well images into horizontal stripes (1 per row)
+    logger.info("Concatenate images into a plate..")
+
+    image_row_data = []
+    for current_plate_row in range(1, 17):
+
+        # concatenate horizontally and vertically
+        well_start_id = ((current_plate_row - 1) * 24) + 0
+        well_end_id = current_plate_row * 24
+        sites_row = cv2.hconcat(image_well_data[well_start_id:well_end_id])
+        image_row_data.append(sites_row)
+
+    # concatenate all the stripes into 1 image
+    plate_image = cv2.vconcat(image_row_data)
+    return plate_image
+
+
+
+def copy_well_images_to_output_folder(temp_folder, output_path, well_list, plate_name, style) +
+
+

Copies all temporary well images into the output folder. +Used for when the scope of the operation is 'well' and we want only the well images to be outputed.

+
    Parameters:
+            temp_folder (Path): The path to the temporary working directory where the well images are currently stored in.
+            output_path (Path): The path to the folder where the images should be copied to.
+            well_list (string list): A list of all the well IDs (e.g. ['A01', 'A02', 'A03', ...]).
+            plate_name (string): The name of the current plate (used to generate the output files' names).
+            style (string): The name of the style used for rendering (used to generate the output files' names).
+
+    Returns:
+            8 bit cv2 image
+
+
+ +Expand source code + +
def copy_well_images_to_output_folder(temp_folder, output_path, well_list, plate_name, style):
+    '''
+    Copies all temporary well images into the output folder.
+    Used for when the scope of the operation is 'well' and we want only the well images to be outputed.
+
+            Parameters:
+                    temp_folder (Path): The path to the temporary working directory where the well images are currently stored in.
+                    output_path (Path): The path to the folder where the images should be copied to.
+                    well_list (string list): A list of all the well IDs (e.g. ['A01', 'A02', 'A03', ...]).
+                    plate_name (string): The name of the current plate (used to generate the output files' names).
+                    style (string): The name of the style used for rendering (used to generate the output files' names).
+
+            Returns:
+                    8 bit cv2 image
+    '''
+
+    print("Putting well images into output folder..")
+
+    for current_well in list(well_list):
+        copyfile(
+            temp_folder + "/wells/well-"+current_well+".png",
+            output_path+'/'+plate_name+"-"+current_well+"-"+style+".png"
+        )
+
+    return
+
+
+
+def generate_multiplexed_well_images(data_df, temp_folder, style, display_well_details, scope) +
+
+

Generates a colorized image from all 5 channels of a well, for all wells, and saves it in the temporary directory.

+
    Parameters:
+            data_df (Pandas DataFrame): Dataframe containing the paths to each channel, of each site, of each well.
+            temp_folder_path (Path): The path to the folder where temporary data can be stored.
+            style (string): The name of the style being used to generate the colorized image.
+            style (string): The name of rendering style.
+            display_well_details (bool): Whether or not the name of the well should be printed on its generated image.
+            scope (string): 'plate' or 'wells' (this will have an impact on the resizing of the well/site images).
+
+    Returns:
+            True (in case of success)
+
+
+ +Expand source code + +
def generate_multiplexed_well_images(
+    data_df, temp_folder, style, display_well_details, scope
+):
+    '''
+    Generates a colorized image from all 5 channels of a well, for all wells, and saves it in the temporary directory.
+
+            Parameters:
+                    data_df (Pandas DataFrame): Dataframe containing the paths to each channel, of each site, of each well.
+                    temp_folder_path (Path): The path to the folder where temporary data can be stored.
+                    style (string): The name of the style being used to generate the colorized image.
+                    style (string): The name of rendering style.
+                    display_well_details (bool): Whether or not the name of the well should be printed on its generated image.
+                    scope (string): 'plate' or 'wells' (this will have an impact on the resizing of the well/site images).
+
+            Returns:
+                    True (in case of success)
+    '''
+
+    if scope == 'wells':
+        rescale_ratio = parameters.rescale_ratio_picasso_wells
+    if scope == 'plate':
+        rescale_ratio = parameters.rescale_ratio_picasso_plate
+
+    # get the well list
+    well_list = list(set(data_df["well"]))
+    well_list.sort()
+
+    # multiplex all well/site channels into 1 well/site 8bit color image
+    wellprogressbar = tqdm(list(well_list), unit="wells", leave=False)
+    for current_well in wellprogressbar:
+        current_well_sites_multiplexed_image_list = []
+        for current_site in range(1, 7):
+
+            # get the image path list for the channels of the site, in the correct order
+            current_sites_df = data_df[
+                ((data_df["well"] == current_well) &
+                 (data_df["site"] == current_site))
+            ]
+            current_sites_df_ordered = current_sites_df.sort_values(
+                by="channel", ascending=True
+            )
+
+            channel_images_path = current_sites_df_ordered["fullpath"].to_list()
+
+            # proceed to the generation using shaker 4 function with a first predefined fingerprint
+            multiplexed_image = colorizer(
+                img_channels_fullpath=channel_images_path,
+                rescale_ratio=rescale_ratio,
+                max_multiply_coef=8,
+                style=style,
+            )
+
+            # collect image in memory
+            current_well_sites_multiplexed_image_list.append(multiplexed_image)
+
+        # save well image
+        sites_row1 = cv2.hconcat(
+            [
+                current_well_sites_multiplexed_image_list[0],
+                current_well_sites_multiplexed_image_list[1],
+                current_well_sites_multiplexed_image_list[2],
+            ]
+        )
+        sites_row2 = cv2.hconcat(
+            [
+                current_well_sites_multiplexed_image_list[3],
+                current_well_sites_multiplexed_image_list[4],
+                current_well_sites_multiplexed_image_list[5],
+            ]
+        )
+        all_sites_image = cv2.vconcat([sites_row1, sites_row2])
+
+        # add fingerprint id on image
+        if display_well_details:
+            text = str(current_well)
+            font = cv2.FONT_HERSHEY_SIMPLEX
+            cv2.putText(
+                img=all_sites_image,
+                text=text,
+                org=(math.ceil(80*rescale_ratio), math.ceil(80*rescale_ratio)),
+                fontFace=font,
+                fontScale=2.2*rescale_ratio,
+                thickness=math.ceil(3*rescale_ratio),
+                color=(192, 192, 192),
+                lineType=cv2.INTER_AREA,
+            )
+        if scope == 'plate':
+            # add well marks on borders
+            image_shape = all_sites_image.shape
+            cv2.rectangle(
+                all_sites_image,
+                (0, 0),
+                (image_shape[1], image_shape[0]),
+                (192, 192, 192),
+                math.ceil(8*rescale_ratio),
+            )
+
+        cv2.imwrite(
+            temp_folder + "/wells/well-" + str(current_well) + ".png",
+            all_sites_image,
+        )
+
+    return
+
+
+
+def get_images_full_path(channel_string_ids, plate_input_path) +
+
+

Finds all the paths to all the channels' images from the input folder

+
    Parameters:
+            channel_string_ids (string list): A list of all the channels IDs to be loaded.
+            plate_input_path (Path): The path to the folder where the input images are stored.
+
+    Returns:
+            Path list: A list of all the paths to all images of each channels
+
+
+ +Expand source code + +
def get_images_full_path(channel_string_ids, plate_input_path):
+    '''
+    Finds all the paths to all the channels' images from the input folder
+
+            Parameters:
+                    channel_string_ids (string list): A list of all the channels IDs to be loaded.
+                    plate_input_path (Path): The path to the folder where the input images are stored.
+
+            Returns:
+                    Path list: A list of all the paths to all images of each channels
+    '''
+
+    # get the files from the plate folder, for the targeted channel
+    images_full_path_list = []
+    for current_channel in channel_string_ids:
+        current_channel_images_full_path_list = list(
+            Path(plate_input_path).glob("*" + current_channel + ".tif")
+        )
+        images_full_path_list = images_full_path_list + \
+            current_channel_images_full_path_list
+
+    # check that we get expected images for a 384 well image
+    try:
+        assert len(images_full_path_list) == 2304 * 5
+    except AssertionError:
+        print(
+            "The plate does not have the exact image count: expected " +
+            str(2304 * 5) + ", got "
+            + str(len(images_full_path_list)),
+        )
+
+    return images_full_path_list
+
+
+
+def picasso_generate_plate_image(source_path, plate_name, output_path, temp_folder_path, style, scope, display_well_details) +
+
+

Generates cell-painted colorized images of individual wells or of a whole plate.

+
    Parameters:
+            source_path (Path): The folder where the input images of the plate are stored.
+            plate_name (string): The name of the plate being rendered.
+            output_path (Path): The path to the folder where the output images should be stored.
+            temp_folder_path (Path): The path to the folder where temporary data can be stored.
+            style (string): The name of the rendering style.
+            scope (string):
+                    Either 'wells' or 'plate'. Defines if we should generate individual well images,
+                    or concatenate them into a single plate image.
+            display_well_details (bool): Whether or not the name of the well should be written on the generated images.
+
+    Returns:
+            8 bit cv2 image(s):
+                    If the scope is 'wells', then all colorized well images are outputed to the output folder.
+                    If the scope is 'wells', then the well images are concatenated into one image of the whole
+                    plate before being outputed to the output folder.
+
+
+ +Expand source code + +
def picasso_generate_plate_image(
+    source_path,
+    plate_name,
+    output_path,
+    temp_folder_path,
+    style,
+    scope,
+    display_well_details,
+):
+    '''
+    Generates cell-painted colorized images of individual wells or of a whole plate.
+
+            Parameters:
+                    source_path (Path): The folder where the input images of the plate are stored.
+                    plate_name (string): The name of the plate being rendered.
+                    output_path (Path): The path to the folder where the output images should be stored.
+                    temp_folder_path (Path): The path to the folder where temporary data can be stored.
+                    style (string): The name of the rendering style.
+                    scope (string):
+                            Either 'wells' or 'plate'. Defines if we should generate individual well images,
+                            or concatenate them into a single plate image.
+                    display_well_details (bool): Whether or not the name of the well should be written on the generated images.
+
+            Returns:
+                    8 bit cv2 image(s):
+                            If the scope is 'wells', then all colorized well images are outputed to the output folder.
+                            If the scope is 'wells', then the well images are concatenated into one image of the whole
+                            plate before being outputed to the output folder.
+    '''
+
+    # define a temp folder for the run
+    temp_folder_path = temp_folder_path + "/tmpgen-" + plate_name + "picasso"
+
+    # remove temp dir if existing
+    shutil.rmtree(temp_folder_path, ignore_errors=True)
+
+    # create the temporary directory structure to work on wells
+    try:
+        os.mkdir(temp_folder_path)
+    except FileExistsError:
+        pass
+    # also create a subfolder to store well images
+    try:
+        os.mkdir(temp_folder_path + "/wells")
+    except FileExistsError:
+        pass
+
+    # read the plate input path
+    plate_input_path = Path(source_path)
+
+    # get the list of all paths for each channel image
+    images_full_path_list = get_images_full_path(
+        # TODO: adapt for less than 5 selected channels?
+        channel_string_ids=["C01", "C02", "C03", "C04", "C05"],
+        plate_input_path=plate_input_path,
+    )
+
+    # build a database of the theorical plate
+    # TODO: adapt for less than 5 selected channels?
+    data_df = build_robustized_plate_dataframe(images_full_path_list)
+
+    # get the well list
+    well_list = list(set(data_df["well"]))
+    well_list.sort()
+
+    # generate images inside the temp folder
+    generate_multiplexed_well_images(
+        data_df=data_df,
+        temp_folder=temp_folder_path,
+        style=style,
+        display_well_details=display_well_details,
+        scope=scope,
+    )
+
+    if scope == 'plate':
+        # concatenate well images into a plate image
+        plate_image = concatenate_well_images(well_list, temp_folder_path)
+
+        # save image
+        plate_image_path = (
+            output_path + "/" + plate_name + "-" +
+            "picasso" + "-" + str(style) + ".jpg"
+        )
+        cv2.imwrite(plate_image_path, plate_image)
+
+        print(" -> Generated image of size:", plate_image.shape)
+        print(" -> Saved as ", plate_image_path)
+
+    if scope == 'wells':
+        # copy well files in output folder
+        copy_well_images_to_output_folder(
+            temp_folder_path, output_path, well_list, plate_name, style)
+
+        print(" -> Saved well images in ", output_path)
+
+    # purge temp files
+    logger.info("Purge temporary folder")
+    shutil.rmtree(temp_folder_path, ignore_errors=True)
+
+    return
+
+
+
+
+
+
+
+ +
+ + + \ No newline at end of file diff --git a/docs/toolbox.html b/docs/toolbox.html new file mode 100644 index 0000000..d55ff96 --- /dev/null +++ b/docs/toolbox.html @@ -0,0 +1,377 @@ + + + + + + +lumos.toolbox API documentation + + + + + + + + + + + +
+
+
+

Module lumos.toolbox

+
+
+

Extra helper functions for lumos.

+
+ +Expand source code + +
#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+'''
+Extra helper functions for lumos.
+'''
+import cv2
+import os
+from . import logger
+
+
+def load_site_image(site, current_wells_df, source_folder):
+    '''
+    Loads a site image, for a specific channel.
+
+            Parameters:
+                    site (int): The id of the site (between 1 and 6).
+                    current_wells_df (DataFrame): The dataframe containing image metadata.
+                    source_folder (Path): The path to the folder where the images are stored.
+
+            Returns:
+                    16-bit cv2 image
+    '''
+
+    for _, current_site in current_wells_df.iterrows():
+        # process field 1
+        if current_site["site"] == site:
+            if not os.path.isfile(source_folder + "/" + str(current_site["filename"])):
+                logger.debug("Path to site image does not exist")
+            site_img = cv2.imread(
+                source_folder + "/" + str(current_site["filename"]), -1
+            )
+
+    try:
+        site_img.shape
+    except:
+        logger.warning("Failed to load site image")
+        return
+
+    return site_img
+
+
+def load_well_image(well, source_folder):
+    '''
+    Loads a well image, for a specific channel.
+    Well images are temporary images made by lumos for a specific channel.
+    They should be found inside of the working temporary directory.
+
+            Parameters:
+                    well (string): The id of the well (e.g. D23).
+                    source_folder (Path): The path to the folder where the images are stored.
+
+            Returns:
+                    16-bit cv2 image
+    '''
+
+    well_image_path = source_folder + "/well-" + str(well) + ".png"
+    if not os.path.isfile(well_image_path):
+        logger.debug("Path to well image does not exist")
+    well_img = cv2.imread(well_image_path)
+
+    try:
+        well_img.shape
+    except:
+        logger.warning("Failed to load well image")
+
+    return well_img
+
+
+def draw_markers(image, color):
+    '''
+    Draws standard markers on an image. This includes highlighted corners and an "empty" symbol in the middle of the image.
+
+            Parameters:
+                    image (cv2 image|np.array): The input image onto which the markers will be drawn.
+                    color (int|int tuple): The intensity/color value that the markers will have.
+
+            Returns:
+                    modified image
+    '''
+
+    length = int(min(image.shape[0], image.shape[1]) / 10)
+    thickness = int(min(image.shape[0], image.shape[1]) / 20)
+
+    startCorner1 = (0, 0)
+    startCorner2 = (image.shape[0], 0)
+    startCorner3 = (0, image.shape[1])
+    startCorner4 = image.shape[:2]
+
+    # draw corner 1
+    image = cv2.line(image, startCorner1, (int(
+        startCorner1[0] + length), startCorner1[1]), color, thickness)
+    image = cv2.line(image, startCorner1, (startCorner1[0], int(
+        startCorner1[1] + length)), color, thickness)
+    # draw corner 2
+    image = cv2.line(image, startCorner2, (int(
+        startCorner2[0] - length), startCorner2[1]), color, thickness)
+    image = cv2.line(image, startCorner2, (startCorner2[0], int(
+        startCorner2[1] + length)), color, thickness)
+    # draw corner 3
+    image = cv2.line(image, startCorner3, (int(
+        startCorner3[0] + length), startCorner3[1]), color, thickness)
+    image = cv2.line(image, startCorner3, (startCorner3[0], int(
+        startCorner3[1] - length)), color, thickness)
+    # draw corner 3
+    image = cv2.line(image, startCorner4, (int(
+        startCorner4[0] - length), startCorner4[1]), color, thickness)
+    image = cv2.line(image, startCorner4, (startCorner4[0], int(
+        startCorner4[1] - length)), color, thickness)
+
+    # draw circle
+    radius = int(min(image.shape[0], image.shape[1]) / 5)
+    center = (int(image.shape[0]/2), int(image.shape[1]/2))
+    image = cv2.circle(image, center, radius, color, thickness)
+
+    # draw cross-line
+    startCrossLine = (center[0]-radius, center[1]+radius)
+    endCrossLine = (center[0]+radius, center[1]-radius)
+    image = cv2.line(image, startCrossLine, endCrossLine, color, thickness)
+
+    # # draw character (cv2 does not support unicode characters)
+    # text = "Ø"
+    # image = cv2.putText(
+    #     img = image,
+    #     text = text,
+    #     org = (int(image.shape[0]/4), int(image.shape[1]*3/4)),
+    #     fontFace = cv2.FONT_HERSHEY_SIMPLEX,
+    #     fontScale = image.shape[0]/50,
+    #     color = color,
+    #     thickness = thickness,
+    # )
+
+    return image
+
+
+
+
+
+
+
+

Functions

+
+
+def draw_markers(image, color) +
+
+

Draws standard markers on an image. This includes highlighted corners and an "empty" symbol in the middle of the image.

+
    Parameters:
+            image (cv2 image|np.array): The input image onto which the markers will be drawn.
+            color (int|int tuple): The intensity/color value that the markers will have.
+
+    Returns:
+            modified image
+
+
+ +Expand source code + +
def draw_markers(image, color):
+    '''
+    Draws standard markers on an image. This includes highlighted corners and an "empty" symbol in the middle of the image.
+
+            Parameters:
+                    image (cv2 image|np.array): The input image onto which the markers will be drawn.
+                    color (int|int tuple): The intensity/color value that the markers will have.
+
+            Returns:
+                    modified image
+    '''
+
+    length = int(min(image.shape[0], image.shape[1]) / 10)
+    thickness = int(min(image.shape[0], image.shape[1]) / 20)
+
+    startCorner1 = (0, 0)
+    startCorner2 = (image.shape[0], 0)
+    startCorner3 = (0, image.shape[1])
+    startCorner4 = image.shape[:2]
+
+    # draw corner 1
+    image = cv2.line(image, startCorner1, (int(
+        startCorner1[0] + length), startCorner1[1]), color, thickness)
+    image = cv2.line(image, startCorner1, (startCorner1[0], int(
+        startCorner1[1] + length)), color, thickness)
+    # draw corner 2
+    image = cv2.line(image, startCorner2, (int(
+        startCorner2[0] - length), startCorner2[1]), color, thickness)
+    image = cv2.line(image, startCorner2, (startCorner2[0], int(
+        startCorner2[1] + length)), color, thickness)
+    # draw corner 3
+    image = cv2.line(image, startCorner3, (int(
+        startCorner3[0] + length), startCorner3[1]), color, thickness)
+    image = cv2.line(image, startCorner3, (startCorner3[0], int(
+        startCorner3[1] - length)), color, thickness)
+    # draw corner 3
+    image = cv2.line(image, startCorner4, (int(
+        startCorner4[0] - length), startCorner4[1]), color, thickness)
+    image = cv2.line(image, startCorner4, (startCorner4[0], int(
+        startCorner4[1] - length)), color, thickness)
+
+    # draw circle
+    radius = int(min(image.shape[0], image.shape[1]) / 5)
+    center = (int(image.shape[0]/2), int(image.shape[1]/2))
+    image = cv2.circle(image, center, radius, color, thickness)
+
+    # draw cross-line
+    startCrossLine = (center[0]-radius, center[1]+radius)
+    endCrossLine = (center[0]+radius, center[1]-radius)
+    image = cv2.line(image, startCrossLine, endCrossLine, color, thickness)
+
+    # # draw character (cv2 does not support unicode characters)
+    # text = "Ø"
+    # image = cv2.putText(
+    #     img = image,
+    #     text = text,
+    #     org = (int(image.shape[0]/4), int(image.shape[1]*3/4)),
+    #     fontFace = cv2.FONT_HERSHEY_SIMPLEX,
+    #     fontScale = image.shape[0]/50,
+    #     color = color,
+    #     thickness = thickness,
+    # )
+
+    return image
+
+
+
+def load_site_image(site, current_wells_df, source_folder) +
+
+

Loads a site image, for a specific channel.

+
    Parameters:
+            site (int): The id of the site (between 1 and 6).
+            current_wells_df (DataFrame): The dataframe containing image metadata.
+            source_folder (Path): The path to the folder where the images are stored.
+
+    Returns:
+            16-bit cv2 image
+
+
+ +Expand source code + +
def load_site_image(site, current_wells_df, source_folder):
+    '''
+    Loads a site image, for a specific channel.
+
+            Parameters:
+                    site (int): The id of the site (between 1 and 6).
+                    current_wells_df (DataFrame): The dataframe containing image metadata.
+                    source_folder (Path): The path to the folder where the images are stored.
+
+            Returns:
+                    16-bit cv2 image
+    '''
+
+    for _, current_site in current_wells_df.iterrows():
+        # process field 1
+        if current_site["site"] == site:
+            if not os.path.isfile(source_folder + "/" + str(current_site["filename"])):
+                logger.debug("Path to site image does not exist")
+            site_img = cv2.imread(
+                source_folder + "/" + str(current_site["filename"]), -1
+            )
+
+    try:
+        site_img.shape
+    except:
+        logger.warning("Failed to load site image")
+        return
+
+    return site_img
+
+
+
+def load_well_image(well, source_folder) +
+
+

Loads a well image, for a specific channel. +Well images are temporary images made by lumos for a specific channel. +They should be found inside of the working temporary directory.

+
    Parameters:
+            well (string): The id of the well (e.g. D23).
+            source_folder (Path): The path to the folder where the images are stored.
+
+    Returns:
+            16-bit cv2 image
+
+
+ +Expand source code + +
def load_well_image(well, source_folder):
+    '''
+    Loads a well image, for a specific channel.
+    Well images are temporary images made by lumos for a specific channel.
+    They should be found inside of the working temporary directory.
+
+            Parameters:
+                    well (string): The id of the well (e.g. D23).
+                    source_folder (Path): The path to the folder where the images are stored.
+
+            Returns:
+                    16-bit cv2 image
+    '''
+
+    well_image_path = source_folder + "/well-" + str(well) + ".png"
+    if not os.path.isfile(well_image_path):
+        logger.debug("Path to well image does not exist")
+    well_img = cv2.imread(well_image_path)
+
+    try:
+        well_img.shape
+    except:
+        logger.warning("Failed to load well image")
+
+    return well_img
+
+
+
+
+
+
+
+ +
+ + + \ No newline at end of file diff --git a/lumos/__init__.py b/lumos/__init__.py index e69de29..fd8be1a 100644 --- a/lumos/__init__.py +++ b/lumos/__init__.py @@ -0,0 +1,3 @@ +''' +Lumos - Back-end module +''' \ No newline at end of file diff --git a/lumos/generator.py b/lumos/generator.py index 3fd99c2..7138806 100644 --- a/lumos/generator.py +++ b/lumos/generator.py @@ -1,56 +1,65 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -""" Main functions to generate platemaps with lumos - -""" +''' +Main functions to generate platemaps with lumos. +''' +import math +import multiprocessing import os -from pathlib import Path, PureWindowsPath +from pathlib import Path +import platform import pandas as pd from shutil import copyfile import shutil from . import toolbox +from . import logger from . import parameters import cv2 from tqdm import tqdm -import logging import numpy as np -def generate_plate_image( +def generate_plate_image_for_channel( plate_input_path_string, plate_name, channel_to_render, channel_label, temp_folder, + keep_temp_files, ): - """Generate an image of a cellpainting plate for a specific channel - Args: - plate_images_folder_path : The folder where the images of the plate are stored - plate_name: name of the plate - channel_to_render: the cellpainting channel to render - channel_label: The text describing the channel type - temp_folder_path: The folder where temporary data can be stored - Return: 8 bit cv2 image - """ + ''' + Generates an image of a cellpainting plate for a specific channel. + + Parameters: + plate_images_folder_path (Path): The path to the folder where the images of the plate are stored. + plate_name (string): Name of the plate. + channel_to_render (string): The cellpainting channel to render. + channel_label (string): The label describing the channel type. + temp_folder_path (Path): The folder where temporary data can be stored. + + Returns: + 8-bit cv2 image + ''' # define a temp folder for the run - temp_folder = temp_folder + "/tmpgen-" + plate_name + channel_to_render + temp_folder = temp_folder + "/lumos-tmpgen-" + plate_name + channel_to_render # remove temp dir if existing - shutil.rmtree(temp_folder, ignore_errors=True) + if not keep_temp_files: + logger.debug("Purge temporary folder before plate generation") + shutil.rmtree(temp_folder, ignore_errors=True) # create the temporary directory structure to work on images try: os.mkdir(temp_folder) os.mkdir(temp_folder + "/wells") - except FileExistsError: pass # read the plate input path - plate_input_path = Path(PureWindowsPath(plate_input_path_string)) + plate_input_path = Path(plate_input_path_string) # get the files from the plate folder, for the targeted channel images_full_path_list = list( @@ -61,15 +70,19 @@ def generate_plate_image( try: assert len(images_full_path_list) == 2304 except AssertionError: - print( - "The plate does not have the exact image count expected", - len(images_full_path_list), + logger.p_print( + "The plate does not have the exact image count: expected 2304, got " + + str(len(images_full_path_list)) + ) + logger.warning( + "The plate does not have the exact image count: expected 2304, got " + + str(len(images_full_path_list)) ) - logging.info( - "Start plate image generation for channel:" + logger.info( + "Start plate image generation for channel: " + str(channel_to_render) - + " " + + " - " + str(channel_label) ) @@ -101,10 +114,10 @@ def generate_plate_image( # get the theoretical well list for 384 well plate well_theoretical_list = [ - l + str(r).zfill(2) for l in "ABCDEFGHIJKLMNOP" for r in range(1, 25) + l + str(r).zfill(2) for l in "ABCDEFGHIJKLMNOP" for r in range(1, 25) # e.g. "A01" ] well_site_theoretical_list = [ - [x, r] for x in well_theoretical_list for r in range(1, 7) + [x, r] for x in well_theoretical_list for r in range(1, 7) # e.g. ["A01", 1] .. ["A01", 6] ] # create the theoretical well dataframe @@ -113,7 +126,7 @@ def generate_plate_image( ) # join the real wells with the theoric ones - theoretical_data_df_joined = theoretical_data_df.merge( + data_df_joined = theoretical_data_df.merge( data_df, left_on=["well", "site"], right_on=["well", "site"], @@ -122,22 +135,23 @@ def generate_plate_image( # log if there is a delta between theory and actual plate wells delta = set(well_theoretical_list) - set(image_well_list) - logging.info("Well Delta " + str(delta)) - - # promote theoretical over actual - data_df = theoretical_data_df_joined + logger.debug("Well Delta " + str(delta)) # get the site images and store them locally - logging.info("Copy sources images in temp folder..") + logger.info("Copying sources images in temp folder..") - for index, current_image in tqdm( - data_df.iterrows(), + copyprogressbar = tqdm( + data_df_joined.iterrows(), + total=len(data_df_joined), desc="Download images to temp", unit="images", - colour="#006464", + colour="blue" if platform.system() == 'Windows' else "#006464", leave=True, - ): - # do not copy if file exists + disable=logger._is_in_parallel, + ) + for _, current_image in copyprogressbar: + + # do not copy if temp file already exists, or if source file doesn't exists if not os.path.isfile(temp_folder + "/" + str(current_image["filename"])): try: copyfile( @@ -145,61 +159,82 @@ def generate_plate_image( temp_folder + "/" + str(current_image["filename"]), ) except TypeError: - logging.warning( - "TypeError:" + # this is thrown when the source file does not exist, or when copyfile() fails + logger.warning( + "TypeError: from " + str(current_image["fullpath"]) + + " to " + str(temp_folder) + "/" + str(current_image["filename"]) ) + else: + logger.debug( + "File already exists in temp folder: " + + temp_folder + "/" + str(current_image["filename"]) + ) - # get the well set - well_list = list(set(data_df["well"])) + logger.info("Copying sources images in temp folder..Done") + + # get the list of all the wells + # We first convert to a set to remove redundant wells (duplicate data because each is represented 6 times, one per site) + well_list = list(set(data_df_joined["well"])) well_list.sort() - logging.info("Generate Well images") + logger.info("Generating well images and storing them in temp dir..") # generate one image per well by concatenation of image sites - wellprogressbar = tqdm(list(well_list), unit="wells", leave=False) + wellprogressbar = tqdm( + well_list, + unit="wells", + colour="magenta" if platform.system() == 'Windows' else "#6464a0", + leave=True, + disable=logger._is_in_parallel, + ) for current_well in wellprogressbar: - wellprogressbar.set_description("Processing well %s" % current_well) - # get the 6 images metadate of the well - current_wells_df = data_df[data_df["well"] == current_well] - # load 6 wells into an image list + # get the 6 images metadata of the well + current_wells_df = data_df_joined.loc[data_df_joined["well"] == current_well] + + # load 6 wells into an image list (if image cannot be opened, e.g. if it is missing or corrupted, replace with a placeholder image) image_list = [] for current_site in range(1, 7): img = toolbox.load_site_image(current_site, current_wells_df, temp_folder) try: - img.shape + # resize the image first to reduce computations + img = cv2.resize( + src=img, + dsize=None, + fx=parameters.rescale_ratio, + fy=parameters.rescale_ratio, + interpolation=cv2.INTER_CUBIC, + ) + # normalize the intensity of each channel by a specific coefficient + img = img * parameters.channel_coefficients[channel_to_render] + # convert to 8 bit + img = img / 256 + img = img.astype("uint8") except: - # create blank file - img = np.full((1000, 1000, 1), 32768, np.uint16) - logging.warning("Missing file in well" + current_well) + # create placeholder image when error + img = np.full( + shape=(int(1000*parameters.rescale_ratio), int(1000*parameters.rescale_ratio), 1), + fill_value=parameters.placeholder_background_intensity, + dtype=np.uint8 + ) + img = toolbox.draw_markers(img, parameters.placeholder_markers_intensity) + logger.warning("Missing or corrupted file in well " + current_well + " (site " + str(current_site) + ")") image_list.append(img) - # clip and rescale each image individualy - rescaled_image_list = [] - for img in image_list: - img_norm = img * parameters.channel_coefficients[channel_to_render] - rescaled_image_list.append(img_norm) - # concatenate horizontally and vertically sites_row1 = cv2.hconcat( - [rescaled_image_list[0], rescaled_image_list[1], rescaled_image_list[2]] + [image_list[0], image_list[1], image_list[2]] ) sites_row2 = cv2.hconcat( - [rescaled_image_list[3], rescaled_image_list[4], rescaled_image_list[5]] + [image_list[3], image_list[4], image_list[5]] ) all_sites_image = cv2.vconcat([sites_row1, sites_row2]) - all_sites_image_norm = all_sites_image - - # convert to 8 bit - # comment the following line to generate interesting images - all_sites_image = all_sites_image_norm / 256 - all_sites_image = all_sites_image.astype("uint8") # add well id on image text = current_well + " " + channel_label @@ -207,11 +242,11 @@ def generate_plate_image( cv2.putText( all_sites_image, text, - (25, 125), + (math.ceil(25*parameters.rescale_ratio), math.ceil(125*parameters.rescale_ratio)), font, - 4, + 4*parameters.rescale_ratio, (192, 192, 192), - 8, + math.ceil(8*parameters.rescale_ratio), cv2.INTER_AREA, ) @@ -221,26 +256,21 @@ def generate_plate_image( all_sites_image, (0, 0), (image_shape[1], image_shape[0]), - (192, 192, 192), - 8, - ) - # resize - all_sites_image_resized = cv2.resize( - src=all_sites_image, - dsize=None, - fx=parameters.rescale_ratio, - fy=parameters.rescale_ratio, - interpolation=cv2.INTER_CUBIC, + color=(192, 192, 192), + thickness=1, ) - # save + + # save the image in the temp folder cv2.imwrite( temp_folder + "/wells/well-" + str(current_well) + ".png", - all_sites_image_resized, + all_sites_image, ) + logger.info("Generating well images and storing them in temp dir..Done") + # load all well images and store images in memory into a list - print("Load well images in memory..") - logging.info("Generate Well images") + logger.p_print("Combining well images into final channel image..") + logger.info("Loading well images from temp dir..") image_well_data = [] for current_well in list(well_list): @@ -250,8 +280,10 @@ def generate_plate_image( ) image_well_data.append(well_image) + logger.info("Loading well images from temp dir..Done") + # concatenate all the well images into horizontal stripes (1 per row) - logging.info("Concatenate images into a plate..") + logger.info("Concatenating well images into a plate..") image_row_data = [] for current_plate_row in range(1, 17): @@ -265,51 +297,59 @@ def generate_plate_image( # concatenate all the stripes into 1 image plate_image = cv2.vconcat(image_row_data) - # purge temp files - logging.info("Purge temporary folder") + logger.info("Concatenating well images into a plate..Done") - shutil.rmtree(temp_folder, ignore_errors=True) + # purge temp files + if not keep_temp_files: + logger.debug("Purge temporary folder after generation") + shutil.rmtree(temp_folder, ignore_errors=True) return plate_image def render_single_channel_plateview( - source_path, plate_name, channel, channel_label, output_path, temp_folder + source_path, plate_name, channel_to_render, channel_label, output_path, temp_folder_path, keep_temp_files ): - """Render 1 image for a specific plate channel - args: - source_path: the source folder containing plate images - plate_name: name of the plate - channel: the code of the channel to render (e.g. C02) - channel_label: the text detail of the channel - output_path: the folder where to save the image - temp_folder: temporary work folder - returns: true in case of success - """ + ''' + Renders 1 image for a specific channel of a plate. + + Parameters: + source_path (Path): The path to the folder where the images of the plate are stored. + plate_name (string): Name of the plate. + channel_to_render (string): The name of the channel to render. + channel_label (string): The label describing the channel type. + output_path (Path): The folder where to save the generated image. + temp_folder_path (Path): The folder where temporary data can be stored. + keep_temp_files (bool): [dev] Whether or not the temporary files should be kept between runs. + + Returns: + True (in case of success) + ''' # generate cv2 image for the channel - plate_image = generate_plate_image( + plate_image = generate_plate_image_for_channel( source_path, plate_name, - channel, + channel_to_render, channel_label, - temp_folder, + temp_folder_path, + keep_temp_files ) + logger.p_print(" -> Generated image of size: " + str(plate_image.shape)) + # save image plate_image_path = ( output_path + "/" + plate_name + "-" - + str(channel) + + str(channel_to_render) + "-" - + str(parameters.channel_coefficients[channel]) + + str(parameters.channel_coefficients[channel_to_render]) + ".jpg" ) - print("Generated image of size:", plate_image.shape) - print("Saved as ", plate_image_path) - cv2.imwrite(plate_image_path, plate_image) + logger.p_print(" -> Saved as " + plate_image_path) return @@ -318,71 +358,157 @@ def render_single_plate_plateview( source_path, plate_name, channel_list, - channel_details_dict, output_path, - temp_folder, + temp_folder_path, + keep_temp_files ): - """Render images (1 per channel) for a specific plate - args: - source_path: the source folder containing plate images - plate_name: name of the plate - channel_list: The list of channels to render - channel_details_dict: channel details stored in a dict - output_path: the folder where to save the images - temp_folder: temporary work folder - returns: true in case of success - """ + ''' + Renders 1 image per channel for a specific plate. + + Parameters: + source_path (Path): The path to the folder where the images of the plate are stored. + plate_name (string): Name of the plate. + channel_list (string list): The list of the channels to render. + output_path (Path): The folder where to save the generated image. + temp_folder_path (Path): The folder where temporary data can be stored. + keep_temp_files (bool): [dev] Whether or not the temporary files should be kept between runs. + + Returns: + True (in case of success) + ''' + for current_channel in tqdm( channel_list, desc="Render plate channels", - unit="channel plateview", - colour="green", - bar_format=None, + unit="channel", + colour="green" if platform.system() == 'Windows' else "#00ff00", ): - print("Generate", current_channel, channel_details_dict[current_channel]) + # get the current channel's label + channel_label = parameters.cellplainting_channels_dict[current_channel] + + logger.p_print(os.linesep) + logger.p_print("Generate " + current_channel + " - " + channel_label + os.linesep) render_single_channel_plateview( source_path, plate_name, current_channel, - channel_details_dict[current_channel], + channel_label, output_path, - temp_folder, + temp_folder_path, + keep_temp_files ) return -def render_single_run_plateview( +def render_single_plate_plateview_parallelism( source_path, - folder_list, + plate_name, channel_list, - channel_details_dict, output_path, - temp_folder, + temp_folder_path, + parallelism, + keep_temp_files ): - """Render images for all plates of a run - args: - source_path: the source folder containing plate images - folder_list: name of the plates and their respective path inside a dict - channel_list: The list of channels to render for each plate - channel_details_dict: channel details stored in a dict - output_path: the folder where to save the images - temp_folder: temporary work folder - returns: true in case of success - """ - - for current_plate in folder_list.keys(): - print("Render", current_plate) + ''' + Renders, in parallel, 1 image per channel for a specific plate. + + Parameters: + source_path (Path): The path to the folder where the images of the plate are stored. + plate_name (string): Name of the plate. + channel_list (string list): The list of the channels to render. + output_path (Path): The folder where to save the generated image. + temp_folder_path (Path): The folder where temporary data can be stored. + parallelism (int): On how many CPU cores should the computation be spread. + keep_temp_files (bool): [dev] Whether or not the temporary files should be kept between runs. + + Returns: + True (in case of success) + ''' + + n_cores = min(parallelism, multiprocessing.cpu_count()) + pool = multiprocessing.Pool(n_cores) + try: + for current_channel in channel_list: + # get the current channel's label + channel_label = parameters.cellplainting_channels_dict[current_channel] + + pool.apply_async(render_single_channel_plateview, args=( + source_path, + plate_name, + current_channel, + channel_label, + output_path, + temp_folder_path, + keep_temp_files + )) + + pool.close() + pool.join() + + except KeyboardInterrupt: + # does not work: this is an issue with the multiprocessing library + pool.terminate() + pool.join() + + return + + +def render_single_run_plateview( + source_folder_dict, + channel_list, + output_path, + temp_folder_path, + parallelism, + keep_temp_files +): + ''' + Renders images for all plates of a run. Compatible with parallelism. + + Parameters: + source_folder_dict (dict): A dictionary of the name of the plates and their respective path. + channel_list (string list): The list of the channels to render for all plates. + output_path (Path): The folder where to save the generated image. + temp_folder_path (Path): The folder where temporary data can be stored. + parallelism (int): On how many CPU cores should the computation be spread. + keep_temp_files (bool): [dev] Whether or not the temporary files should be kept between runs. + + Returns: + True (in case of success) + ''' + runprogressbar = tqdm( + source_folder_dict.keys(), + total=len(source_folder_dict), + desc="Run progress", + unit="plates", + colour='cyan' if platform.system() == 'Windows' else "#0AAFAF", + leave=True, + ) + for current_plate in runprogressbar: # render all the channels of the plate - render_single_plate_plateview( - folder_list[current_plate], - current_plate, - parameters.default_per_plate_channel_to_render, - parameters.cellplainting_channels_dict, - output_path, - temp_folder, - ) + if parallelism == 1: + render_single_plate_plateview( + source_folder_dict[current_plate], + current_plate, + channel_list, + output_path, + temp_folder_path, + keep_temp_files, + ) + else: + render_single_plate_plateview_parallelism( + source_folder_dict[current_plate], + current_plate, + channel_list, + output_path, + temp_folder_path, + parallelism, + keep_temp_files, + ) + + print(os.linesep + os.linesep + "Run completed!") + print(str(len(source_folder_dict.keys())), "plate(s) have been processed.", os.linesep) return diff --git a/lumos/logger.py b/lumos/logger.py new file mode 100644 index 0000000..d7b5f71 --- /dev/null +++ b/lumos/logger.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +''' +Logger functions for lumos. +''' +from logging.handlers import RotatingFileHandler +import logging +import os + + +# state variable to know if the current lumos session is using parallelism +_is_in_parallel = True + + +def setup(temp_directory, is_in_parallel): + ''' + Sets-up the logger inside of the working temporary directory. If parallelism is enabled, no logs are stored (they will be printed to the console by default). + + Parameters: + temp_directory (Path): The path to the temporary directory. + parallelism (bool): Whether or not parallel computation is enabled for the current run of the program. + ''' + + # set the global state variable of the module + global _is_in_parallel + _is_in_parallel = is_in_parallel + + if not _is_in_parallel: + + # define log format + log_formatter = logging.Formatter('%(asctime)s %(levelname)s:\t%(message)s') + + # create logger + app_log = logging.getLogger('root') + + # create a rotating log file for regular execution (3 files * 2MB max) + my_handler = RotatingFileHandler( + temp_directory + "/lumos.log", + mode='a', + maxBytes=2*1024*1024, + backupCount=2, + encoding=None, + delay=0 + ) + + my_handler.setFormatter(log_formatter) + my_handler.setLevel(logging.DEBUG) + + app_log.setLevel(logging.DEBUG) + app_log.addHandler(my_handler) + + else: + # don't log anything, as it is not compatible with multiprocessing + pass + + +def p_print(text, end=os.linesep): + ''' + Parallel print: Handles printing to the console, according to if parallelism is being used or not. + + Parameters: + text (string): The text to be printed. + end (string): What the separating character at the end of the print should be. + ''' + if not _is_in_parallel: + print(text, end=end) + + +def debug(text): + ''' + Stores the message as a DEBUG log in the log file (according to if parallelism is being used or not). + + Parameters: + text (string): The message to be stored. + ''' + if not _is_in_parallel: + logging.getLogger('root').debug(text) + + +def info(text): + ''' + Stores the message as an INFO log in the log file (according to if parallelism is being used or not). + + Parameters: + text (string): The message to be stored. + ''' + if not _is_in_parallel: + logging.getLogger('root').info(text) + + +def warning(text): + ''' + Stores the message as a WARNING log in the log file (according to if parallelism is being used or not). + + Parameters: + text (string): The message to be stored. + ''' + if not _is_in_parallel: + logging.getLogger('root').warning(text) + + +def error(text): + ''' + Stores the message as an ERROR log in the log file (according to if parallelism is being used or not). + + Parameters: + text (string): The message to be stored. + ''' + if not _is_in_parallel: + logging.getLogger('root').error(text) + + +def critical(text): + ''' + Stores the message as a CRITICAL log in the log file (according to if parallelism is being used or not). + + Parameters: + text (string): The message to be stored. + ''' + if not _is_in_parallel: + logging.getLogger('root').critical(text) diff --git a/lumos/parameters.py b/lumos/parameters.py index 54325f2..05ddce5 100644 --- a/lumos/parameters.py +++ b/lumos/parameters.py @@ -1,16 +1,25 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -cellplainting_channels = [ - ["C01 DNA Hoechst 33342", "C01"], - ["C02 ER Concanavalin A", "C02"], - ["C03 RNA SYTO 14", "C03"], - ["C04 AGP Phalloidin and WGA", "C04"], - ["C05 MITO MitoTracker", "C05"], - ["C06 Brigtfield depth1", "Z01C06"], - ["C06 Brigtfield depth2", "Z02C06"], - ["C06 Brigtfield depth3", "Z03C06"], +''' +Main parameters for lumos operation. +''' + +cellplainting_channels_info = [ + # Channel number, Channel name, EX wavelength, RGB equivalence + ["C01", "C01 DNA Hoechst 33342", 450, [0, 70, 255]], # ~blue + ["C02", "C02 ER Concanavalin A", 510, [0, 255, 0]], # ~green + ["C03", "C03 RNA SYTO 14", 570, [225, 255, 0]], # ~yellow + ["C04", "C04 AGP Phalloidin and WGA", 630, [255, 79, 0]], # ~orange + ["C05", "C05 MITO MitoTracker Deep Red", 660, [255, 0, 0]], # ~red + ["Z01C06", "C06 Brigtfield depth1", None, None], + ["Z02C06", "C06 Brigtfield depth2", None, None], + ["Z03C06", "C06 Brigtfield depth3", None, None], ] +''' +Matrix of information for each of the channels. + Columns: [Channel number, Channel name, EX wavelength, RGB equivalence] +''' cellplainting_channels_dict = { "C01": "DNA Hoechst 33342", @@ -22,23 +31,23 @@ "Z02C06": "Brightfield depth2", "Z03C06": "Brightfield depth3", } +''' +Dictionary of the channel names and their respective labels. +''' -# what is the default channel to render for a single channel rendering -default_channel_to_render = cellplainting_channels[0][1] - -# what are the channel to render a singleplate rendering -default_per_plate_channel_to_render = [ +# what are the default channels to render for a plate/run +default_channels_to_render = [ "C01", "C02", "C03", "C04", "C05", - "Z01C06", - "Z02C06", - "Z03C06", ] +''' +List of the default channels to render in a run or single plate rendering. +''' -# rescale coefficient factors per channel +# intensity normalizing coefficient factors per channel channel_coefficients = { "C01": 16, "C02": 8, @@ -49,9 +58,58 @@ "Z02C06": 8, "Z03C06": 8, } +''' +Intensity multiplier coefficients for each of the channels (those are arbitrary and used to make interest points easier to see). +''' clipping_threshold_min_value = 1 clipping_threshold_max_value = 12000 normalize_alpha = 0 normalize_beta = 65535 rescale_ratio = 0.1 + +placeholder_background_intensity = 64 +placeholder_markers_intensity = 0 + + +# -------- PARAMETERS ONLY FOR CELL-PAINTING (PICASSO) -------- + +rescale_ratio_picasso_wells = 1 +rescale_ratio_picasso_plate = 0.25 + +# list of merge rendering styles for picasso +fingerprint_style_dict = { + 'accurate': [[], [], []], + 'random': [[], [], []], + 'blueish': [[6, 5, 6, 6, 6], [2, 3, 4, 1, 0], [0, 2, 1]], + 'blueish2': [[4, 6, 5, 5, 7], [3, 2, 0, 4, 1], [0, 1, 2]], + 'blueredgreen': [[3, 8, 4, 4, 8], [0, 3, 4, 2, 1], [2, 0, 1]], + 'blueredgreen2': [[3, 4, 4, 5, 6], [2, 3, 4, 1, 0], [2, 1, 0]], + 'blueredgreen3': [[8, 4, 6, 5, 8], [1, 3, 4, 2, 0], [0, 1, 2]], + 'reddish': [[7, 7, 4, 4, 1], [2, 1, 3, 4, 0], [1, 0, 2]], + 'reddish2': [[7, 3, 6, 8, 5], [1, 2, 3, 0, 4], [1, 0, 2]], + 'purple': [[2, 6, 6, 7, 2], [3, 1, 2, 4, 0], [0, 1, 2]], + 'purple2': [[1, 7, 8, 6, 8], [2, 4, 0, 3, 1], [0, 1, 2]], + 'chthulu': [[3, 2, 3, 5, 7], [0, 3, 2, 1, 4], [1, 0, 2]], + 'meduse': [[8, 8, 3, 7, 8], [0, 3, 4, 1, 2], [2, 0, 1]], + 'alien': [[3, 6, 4, 3, 3], [1, 3, 2, 4, 0], [1, 0, 2]], +} +''' +Dictionary of the styles that can be used for "merge cell painting", and their associated coefficients. +''' + +accurate_style_parameters = { + 'intensity': [11, 9, 2, 4, 12], + 'contrast': [0, 0, 0.5, 1, 1.85], +} +''' +Dictionary of the coefficients used for "accurate" cell painting (using an approximation of the actual colors of the channels emitted wavelengths) +''' + +# Other styles not integrated +# 'darkgreenblue': [[3,6,2,2,8],[2,1,3,0,4],[2,0,1]], +# 'fingerprint1': [[6,2,3,3,2],[3,1,0,4,2],[0,1,2]], +# 'fingerprint3': [[7,7,6,4,3],[0,2,1,3,4],[1,0,2]], +# 'fingerprint2': [[2,6,6,1,5],[4,3,1,2,0],[2,0,1]], +# 'fingerprint6': [[8,7,4,4,7],[1,3,2,4,0],[0,1,2]], +# 'fingerprint8': [[2,1,4,6,5],[0,3,2,4,1],[0,2,1]], diff --git a/lumos/picasso.py b/lumos/picasso.py new file mode 100644 index 0000000..383bbeb --- /dev/null +++ b/lumos/picasso.py @@ -0,0 +1,575 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +''' +Main functions to generate cell-painted platemaps with Lumos Picasso. +''' + +import math +import os +from pathlib import Path +import pandas as pd +from shutil import copyfile +import shutil +from . import toolbox +from . import parameters +import cv2 +from tqdm import tqdm +from . import logger +import numpy as np +import random + + +def colorizer( + img_channels_fullpath, + rescale_ratio, + style, + max_multiply_coef=1, + display_fingerprint=False, +): + ''' + Merges input images from different channels into one RGB image. + + Parameters: + img_channels_fullpath (Path list): The list of paths to the channels' images (in proper order [C01,C02,C03,C04,C05]). + rescale_ratio (float): The ratio used to rescale the image before generation. + style (string): The name of the style being used to generate the colorized image. + max_multiply_coef (int): Max multiplication factor in case of random coefficient generation. + display_fingerprint (bool): Whether the coefficients used for generation should be printed on the output image. + + Returns: + 8-bit cv2 image + ''' + + # load images from path list + resize + convert to 8bit + np_image_channels_array = [] + for current_image in img_channels_fullpath: + # load image + img16 = cv2.imread(str(current_image), -1) + try: + assert(not img16.shape == (0, 0)) + except: + # create blank file + img16 = np.full(shape=(1000, 1000, 1), + fill_value=0, dtype=np.uint16) + logger.warning("Missing or corrupted image " + str(current_image)) + + # resize image + img16 = cv2.resize( + src=img16, + dsize=None, + fx=rescale_ratio, + fy=rescale_ratio, + interpolation=cv2.INTER_CUBIC, + ) + + # convert image to 8bit + img8 = (img16 / 256).astype("uint8") + np_image_channels_array.append(img8) + + # Perform merging, according to the style + if style == 'accurate': + # initialize RGB channels + red_channel = np.zeros(np_image_channels_array[0].shape) + green_channel = np.zeros(np_image_channels_array[0].shape) + blue_channel = np.zeros(np_image_channels_array[0].shape) + + # # compute the mean of each layer + # means=[] + # for idx in range(5): + # means.append(np.mean(np_image_channels_array[idx])) + + # # contrast image at the mean of each layer (naive approche) + # for idx in range(5): + # vLambda = np.vectorize(lambda x : ((x**parameters.accurate_style_parameters['contrast_coeffs'][idx])/means[idx]) if parameters.accurate_style_parameters['contrast_coeffs'][idx] != 0 else x) + # np_image_channels_array[idx] = vLambda(np_image_channels_array[idx]) + + # # perform thresholding at the mean of each layer + # for idx in range(5): + # # thresholder = lambda x : x if x > (means[idx] * parameters.accurate_style_parameters['threshold_coeffs'][idx]) else 0 + # # vThreshold = np.vectorize(thresholder) + # # np_image_channels_array[idx] = vThreshold(np_image_channels_array[idx]) + + # get the current style's contrast coefficients + contrast_coef = parameters.accurate_style_parameters['contrast'] + # add contrast to each layer according to coefficients + for idx in range(5): + contrast = contrast_coef[idx] + f = float(131 * (contrast + 127)) / (127 * (131 - contrast)) + alpha_c = f + gamma_c = 127*(1-f) + np_image_channels_array[idx] = cv2.addWeighted( + np_image_channels_array[idx], alpha_c, np_image_channels_array[idx], 0, gamma_c) + + # get the current style's intensity coefficients + intensity_coef = parameters.accurate_style_parameters['intensity'] + # multiply the intensity of the channels using input coefs + np_image_array_adjusted = [] + for idx in range(5): # TODO: adapt for less than 5 selected channels + np_image_array_adjusted.append( + np_image_channels_array[idx] * intensity_coef[idx]) + np_image_channels_array = np_image_array_adjusted + + # combine the images according to their RGB coefficients + for idx in range(5): + red_channel = red_channel + \ + (np_image_channels_array[idx] / 255 * + parameters.cellplainting_channels_info[idx][3][0]) + green_channel = green_channel + \ + (np_image_channels_array[idx] / 255 * + parameters.cellplainting_channels_info[idx][3][1]) + blue_channel = blue_channel + \ + (np_image_channels_array[idx] / 255 * + parameters.cellplainting_channels_info[idx][3][2]) + + # merge the Blue, Green and Red channels to form the final image + merged_img = cv2.merge( + (blue_channel, green_channel, red_channel) + ) + + else: + + # get the current style's intensity coefficients + intensity_coef = parameters.fingerprint_style_dict[style][0] + + # get other parameters + channel_order = parameters.fingerprint_style_dict[style][1] + target_rgb = parameters.fingerprint_style_dict[style][2] + + # randomly initiate coefficients if they are missing + if len(intensity_coef) == 0 and len(channel_order) == 0 and len(target_rgb) == 0: + # parameters for each channel + intensity_coef = [random.randint( + 1, max_multiply_coef) for x in range(5)] + channel_order = [0, 1, 2, 3, 4] + random.shuffle(channel_order) + target_rgb = [0, 1, 2] + random.shuffle(target_rgb) + + # multiply the intensity of the channels using input coefs + np_image_array_adjusted = [] + # TODO: adapt for less than 5 selected channels? + for index, current_coef_mult in enumerate(intensity_coef): + np_image_array_adjusted.append( + np_image_channels_array[index] * current_coef_mult) + np_image_channels_array = np_image_array_adjusted + + # merge 2 extra channels each on 1 rgb channel + np_image_channels_array[target_rgb[0]] = ( + np_image_channels_array[target_rgb[0]] + + np_image_channels_array[channel_order[3]] + ) + np_image_channels_array[target_rgb[1]] = ( + np_image_channels_array[target_rgb[1]] + + np_image_channels_array[channel_order[4]] + ) + + merged_img = cv2.merge( + ( + np_image_channels_array[channel_order[0]], + np_image_channels_array[channel_order[1]], + np_image_channels_array[channel_order[2]], + ) + ) + + # add fingerprint id on image + if display_fingerprint: + text = str(intensity_coef) + str(channel_order) + str( + target_rgb) if style != 'accurate' else str(parameters.accurate_style_parameters) + font = cv2.FONT_HERSHEY_SIMPLEX + cv2.putText( + merged_img, + text, + (math.ceil(10*rescale_ratio), math.ceil(990*rescale_ratio)), + font, + 0.8*rescale_ratio, + (192, 192, 192), + math.ceil(2*rescale_ratio), + cv2.INTER_AREA, + ) + + return merged_img + + +def generate_multiplexed_well_images( + data_df, temp_folder, style, display_well_details, scope +): + ''' + Generates a colorized image from all 5 channels of a well, for all wells, and saves it in the temporary directory. + + Parameters: + data_df (Pandas DataFrame): Dataframe containing the paths to each channel, of each site, of each well. + temp_folder_path (Path): The path to the folder where temporary data can be stored. + style (string): The name of the style being used to generate the colorized image. + style (string): The name of rendering style. + display_well_details (bool): Whether or not the name of the well should be printed on its generated image. + scope (string): 'plate' or 'wells' (this will have an impact on the resizing of the well/site images). + + Returns: + True (in case of success) + ''' + + if scope == 'wells': + rescale_ratio = parameters.rescale_ratio_picasso_wells + if scope == 'plate': + rescale_ratio = parameters.rescale_ratio_picasso_plate + + # get the well list + well_list = list(set(data_df["well"])) + well_list.sort() + + # multiplex all well/site channels into 1 well/site 8bit color image + wellprogressbar = tqdm(list(well_list), unit="wells", leave=False) + for current_well in wellprogressbar: + current_well_sites_multiplexed_image_list = [] + for current_site in range(1, 7): + + # get the image path list for the channels of the site, in the correct order + current_sites_df = data_df[ + ((data_df["well"] == current_well) & + (data_df["site"] == current_site)) + ] + current_sites_df_ordered = current_sites_df.sort_values( + by="channel", ascending=True + ) + + channel_images_path = current_sites_df_ordered["fullpath"].to_list() + + # proceed to the generation using shaker 4 function with a first predefined fingerprint + multiplexed_image = colorizer( + img_channels_fullpath=channel_images_path, + rescale_ratio=rescale_ratio, + max_multiply_coef=8, + style=style, + ) + + # collect image in memory + current_well_sites_multiplexed_image_list.append(multiplexed_image) + + # save well image + sites_row1 = cv2.hconcat( + [ + current_well_sites_multiplexed_image_list[0], + current_well_sites_multiplexed_image_list[1], + current_well_sites_multiplexed_image_list[2], + ] + ) + sites_row2 = cv2.hconcat( + [ + current_well_sites_multiplexed_image_list[3], + current_well_sites_multiplexed_image_list[4], + current_well_sites_multiplexed_image_list[5], + ] + ) + all_sites_image = cv2.vconcat([sites_row1, sites_row2]) + + # add fingerprint id on image + if display_well_details: + text = str(current_well) + font = cv2.FONT_HERSHEY_SIMPLEX + cv2.putText( + img=all_sites_image, + text=text, + org=(math.ceil(80*rescale_ratio), math.ceil(80*rescale_ratio)), + fontFace=font, + fontScale=2.2*rescale_ratio, + thickness=math.ceil(3*rescale_ratio), + color=(192, 192, 192), + lineType=cv2.INTER_AREA, + ) + if scope == 'plate': + # add well marks on borders + image_shape = all_sites_image.shape + cv2.rectangle( + all_sites_image, + (0, 0), + (image_shape[1], image_shape[0]), + (192, 192, 192), + math.ceil(8*rescale_ratio), + ) + + cv2.imwrite( + temp_folder + "/wells/well-" + str(current_well) + ".png", + all_sites_image, + ) + + return + + +def concatenate_well_images(well_list, temp_folder_path): + ''' + Loads all temporary well images from the temporary directory and concatenates them into one image of the whole plate. + + Parameters: + well_list (string list): A list of all the well IDs (e.g. ['A01', 'A02', 'A03', ...]). + temp_folder_path (Path): The path to the folder where temporary data can be stored. + + Returns: + 8-bit cv2 image: The concatenated image of all the wells + ''' + + # load all well images and store images in memory into a list + print("Load well images in memory..") + logger.info("Load well images in memory..") + + image_well_data = [] + for current_well in list(well_list): + well_image = toolbox.load_well_image( + current_well, + temp_folder_path + "/wells", + ) + image_well_data.append(well_image) + + # concatenate all the well images into horizontal stripes (1 per row) + logger.info("Concatenate images into a plate..") + + image_row_data = [] + for current_plate_row in range(1, 17): + + # concatenate horizontally and vertically + well_start_id = ((current_plate_row - 1) * 24) + 0 + well_end_id = current_plate_row * 24 + sites_row = cv2.hconcat(image_well_data[well_start_id:well_end_id]) + image_row_data.append(sites_row) + + # concatenate all the stripes into 1 image + plate_image = cv2.vconcat(image_row_data) + return plate_image + + +def get_images_full_path(channel_string_ids, plate_input_path): + ''' + Finds all the paths to all the channels' images from the input folder + + Parameters: + channel_string_ids (string list): A list of all the channels IDs to be loaded. + plate_input_path (Path): The path to the folder where the input images are stored. + + Returns: + Path list: A list of all the paths to all images of each channels + ''' + + # get the files from the plate folder, for the targeted channel + images_full_path_list = [] + for current_channel in channel_string_ids: + current_channel_images_full_path_list = list( + Path(plate_input_path).glob("*" + current_channel + ".tif") + ) + images_full_path_list = images_full_path_list + \ + current_channel_images_full_path_list + + # check that we get expected images for a 384 well image + try: + assert len(images_full_path_list) == 2304 * 5 + except AssertionError: + print( + "The plate does not have the exact image count: expected " + + str(2304 * 5) + ", got " + + str(len(images_full_path_list)), + ) + + return images_full_path_list + + +def build_robustized_plate_dataframe(images_full_path_list): + ''' + Scans the input list of Paths to map it to the expected plate structure. + Missing images or wells must be taken into account in the final render. + + Parameters: + images_full_path_list (Path list): A list of all the paths to all the images to be included in the render. + + Returns: + Pandas DataFrame: + A database of all the image paths to each channel, of each site, of each well. + Its columns are: ["well", "site", "channel", "filename", "fullpath"]. + ''' + + # get the filenames list + images_full_path_list.sort() + images_filename_list = [str(x.name) for x in images_full_path_list] + + # get the well list + image_well_list = [x.split("_")[1].split("_T")[0] + for x in images_filename_list] + + # get the siteid list (sitesid from 1 to 6) + image_site_list = [ + x.split("_T0001F")[1].split("L")[0] for x in images_filename_list + ] + image_site_list_int = [int(x) for x in image_site_list] + + # get the channel id list (channel id from 1 to 5) + image_channel_list = [x.split(".ti")[0][-2:] for x in images_filename_list] + image_channel_list_int = [int(x) for x in image_channel_list] + + # zip all in a data structure + image_data_zip = zip( + image_well_list, + image_site_list_int, + image_channel_list_int, + images_filename_list, + images_full_path_list, + ) + + # convert the zip into dataframe + data_df = pd.DataFrame( + list(image_data_zip), + columns=["well", "site", "channel", "filename", "fullpath"], + ) + + # get the theoretical well list for 384 well plate + well_theoretical_list = [ + l + str(r).zfill(2) for l in "ABCDEFGHIJKLMNOP" for r in range(1, 25) + ] + well_channel_theoretical_list = [ + [x, r, c] for x in well_theoretical_list for r in range(1, 7) for c in range(1, 6) + ] + + # create the theoretical well dataframe + theoretical_data_df = pd.DataFrame( + well_channel_theoretical_list, columns=["well", "site", "channel"] + ) + + # join the real wells with the theoric ones + theoretical_data_df_joined = theoretical_data_df.merge( + data_df, + left_on=["well", "site", "channel"], + right_on=["well", "site", "channel"], + how="left", + ) + + # log if there is a delta between theory and actual plate wells + delta = set(well_theoretical_list) - set(image_well_list) + logger.info("Well Delta " + str(delta)) + + return theoretical_data_df_joined + + +def copy_well_images_to_output_folder(temp_folder, output_path, well_list, plate_name, style): + ''' + Copies all temporary well images into the output folder. + Used for when the scope of the operation is 'well' and we want only the well images to be outputed. + + Parameters: + temp_folder (Path): The path to the temporary working directory where the well images are currently stored in. + output_path (Path): The path to the folder where the images should be copied to. + well_list (string list): A list of all the well IDs (e.g. ['A01', 'A02', 'A03', ...]). + plate_name (string): The name of the current plate (used to generate the output files' names). + style (string): The name of the style used for rendering (used to generate the output files' names). + + Returns: + 8 bit cv2 image + ''' + + print("Putting well images into output folder..") + + for current_well in list(well_list): + copyfile( + temp_folder + "/wells/well-"+current_well+".png", + output_path+'/'+plate_name+"-"+current_well+"-"+style+".png" + ) + + return + + +def picasso_generate_plate_image( + source_path, + plate_name, + output_path, + temp_folder_path, + style, + scope, + display_well_details, +): + ''' + Generates cell-painted colorized images of individual wells or of a whole plate. + + Parameters: + source_path (Path): The folder where the input images of the plate are stored. + plate_name (string): The name of the plate being rendered. + output_path (Path): The path to the folder where the output images should be stored. + temp_folder_path (Path): The path to the folder where temporary data can be stored. + style (string): The name of the rendering style. + scope (string): + Either 'wells' or 'plate'. Defines if we should generate individual well images, + or concatenate them into a single plate image. + display_well_details (bool): Whether or not the name of the well should be written on the generated images. + + Returns: + 8 bit cv2 image(s): + If the scope is 'wells', then all colorized well images are outputed to the output folder. + If the scope is 'wells', then the well images are concatenated into one image of the whole + plate before being outputed to the output folder. + ''' + + # define a temp folder for the run + temp_folder_path = temp_folder_path + "/tmpgen-" + plate_name + "picasso" + + # remove temp dir if existing + shutil.rmtree(temp_folder_path, ignore_errors=True) + + # create the temporary directory structure to work on wells + try: + os.mkdir(temp_folder_path) + except FileExistsError: + pass + # also create a subfolder to store well images + try: + os.mkdir(temp_folder_path + "/wells") + except FileExistsError: + pass + + # read the plate input path + plate_input_path = Path(source_path) + + # get the list of all paths for each channel image + images_full_path_list = get_images_full_path( + # TODO: adapt for less than 5 selected channels? + channel_string_ids=["C01", "C02", "C03", "C04", "C05"], + plate_input_path=plate_input_path, + ) + + # build a database of the theorical plate + # TODO: adapt for less than 5 selected channels? + data_df = build_robustized_plate_dataframe(images_full_path_list) + + # get the well list + well_list = list(set(data_df["well"])) + well_list.sort() + + # generate images inside the temp folder + generate_multiplexed_well_images( + data_df=data_df, + temp_folder=temp_folder_path, + style=style, + display_well_details=display_well_details, + scope=scope, + ) + + if scope == 'plate': + # concatenate well images into a plate image + plate_image = concatenate_well_images(well_list, temp_folder_path) + + # save image + plate_image_path = ( + output_path + "/" + plate_name + "-" + + "picasso" + "-" + str(style) + ".jpg" + ) + cv2.imwrite(plate_image_path, plate_image) + + print(" -> Generated image of size:", plate_image.shape) + print(" -> Saved as ", plate_image_path) + + if scope == 'wells': + # copy well files in output folder + copy_well_images_to_output_folder( + temp_folder_path, output_path, well_list, plate_name, style) + + print(" -> Saved well images in ", output_path) + + # purge temp files + logger.info("Purge temporary folder") + shutil.rmtree(temp_folder_path, ignore_errors=True) + + return diff --git a/lumos/toolbox.py b/lumos/toolbox.py index 546dabb..13e2b67 100644 --- a/lumos/toolbox.py +++ b/lumos/toolbox.py @@ -1,29 +1,32 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -"""Extra helper functions for lumos -""" +''' +Extra helper functions for lumos. +''' import cv2 -import pandas as pd import os -import logging +from . import logger def load_site_image(site, current_wells_df, source_folder): - """load a site image, for a specific channel - Args: - site: the id of the site (between 1 and 6) - current_wells_df: the dataframe containing image metadata - source_folder: the path where the images are stored - Return: - a cv2 image - """ - - for id, current_site in current_wells_df.iterrows(): + ''' + Loads a site image, for a specific channel. + + Parameters: + site (int): The id of the site (between 1 and 6). + current_wells_df (DataFrame): The dataframe containing image metadata. + source_folder (Path): The path to the folder where the images are stored. + + Returns: + 16-bit cv2 image + ''' + + for _, current_site in current_wells_df.iterrows(): # process field 1 if current_site["site"] == site: if not os.path.isfile(source_folder + "/" + str(current_site["filename"])): - logging.info("warning,: path does not exist") + logger.debug("Path to site image does not exist") site_img = cv2.imread( source_folder + "/" + str(current_site["filename"]), -1 ) @@ -31,30 +34,100 @@ def load_site_image(site, current_wells_df, source_folder): try: site_img.shape except: - logging.info("Load Error") + logger.warning("Failed to load site image") return return site_img def load_well_image(well, source_folder): - """load a wellimage, for a specific channel - Well images are temporary images made by lumos for a specific channel - Args: - well: the id of the well (e.g. A23) - source_folder: the path where the images are stored - - Return: - a cv2 image - """ + ''' + Loads a well image, for a specific channel. + Well images are temporary images made by lumos for a specific channel. + They should be found inside of the working temporary directory. + + Parameters: + well (string): The id of the well (e.g. D23). + source_folder (Path): The path to the folder where the images are stored. + + Returns: + 16-bit cv2 image + ''' + well_image_path = source_folder + "/well-" + str(well) + ".png" if not os.path.isfile(well_image_path): - logging.info("warning,: path does not exist") + logger.debug("Path to well image does not exist") well_img = cv2.imread(well_image_path) try: well_img.shape except: - logging.info("Load Error") + logger.warning("Failed to load well image") return well_img + + +def draw_markers(image, color): + ''' + Draws standard markers on an image. This includes highlighted corners and an "empty" symbol in the middle of the image. + + Parameters: + image (cv2 image|np.array): The input image onto which the markers will be drawn. + color (int|int tuple): The intensity/color value that the markers will have. + + Returns: + modified image + ''' + + length = int(min(image.shape[0], image.shape[1]) / 10) + thickness = int(min(image.shape[0], image.shape[1]) / 20) + + startCorner1 = (0, 0) + startCorner2 = (image.shape[0], 0) + startCorner3 = (0, image.shape[1]) + startCorner4 = image.shape[:2] + + # draw corner 1 + image = cv2.line(image, startCorner1, (int( + startCorner1[0] + length), startCorner1[1]), color, thickness) + image = cv2.line(image, startCorner1, (startCorner1[0], int( + startCorner1[1] + length)), color, thickness) + # draw corner 2 + image = cv2.line(image, startCorner2, (int( + startCorner2[0] - length), startCorner2[1]), color, thickness) + image = cv2.line(image, startCorner2, (startCorner2[0], int( + startCorner2[1] + length)), color, thickness) + # draw corner 3 + image = cv2.line(image, startCorner3, (int( + startCorner3[0] + length), startCorner3[1]), color, thickness) + image = cv2.line(image, startCorner3, (startCorner3[0], int( + startCorner3[1] - length)), color, thickness) + # draw corner 3 + image = cv2.line(image, startCorner4, (int( + startCorner4[0] - length), startCorner4[1]), color, thickness) + image = cv2.line(image, startCorner4, (startCorner4[0], int( + startCorner4[1] - length)), color, thickness) + + # draw circle + radius = int(min(image.shape[0], image.shape[1]) / 5) + center = (int(image.shape[0]/2), int(image.shape[1]/2)) + image = cv2.circle(image, center, radius, color, thickness) + + # draw cross-line + startCrossLine = (center[0]-radius, center[1]+radius) + endCrossLine = (center[0]+radius, center[1]-radius) + image = cv2.line(image, startCrossLine, endCrossLine, color, thickness) + + # # draw character (cv2 does not support unicode characters) + # text = "Ø" + # image = cv2.putText( + # img = image, + # text = text, + # org = (int(image.shape[0]/4), int(image.shape[1]*3/4)), + # fontFace = cv2.FONT_HERSHEY_SIMPLEX, + # fontScale = image.shape[0]/50, + # color = color, + # thickness = thickness, + # ) + + return image diff --git a/lumoscli.py b/lumoscli.py index 43a6f7b..b84e731 100644 --- a/lumoscli.py +++ b/lumoscli.py @@ -1,98 +1,167 @@ -"""Lumos command line interface module""" +''' +Lumos command line interface module +''' import tempfile from pathlib import Path import os import fnmatch -import logging import click from art import text2art + from lumos import parameters +from lumos import toolbox +from lumos import logger from lumos.generator import ( render_single_channel_plateview, render_single_plate_plateview, render_single_run_plateview, + render_single_plate_plateview_parallelism, ) +from lumos.picasso import picasso_generate_plate_image +# find the OS temporary directory location +default_temp_directory = tempfile.gettempdir() -# print lumos header -header_ascii_art = text2art("Lumos", font="big") -print(header_ascii_art) +# collect parameters +cellpainting_channels_list = [x[0] for x in parameters.cellplainting_channels_info] +fingerprint_style_list = [x for x in parameters.fingerprint_style_dict] -# initialize a temp dir location -default_temp_directory = tempfile.gettempdir() -# activate log in temporary directory -logging.basicConfig( - filename=default_temp_directory +"lumos.log", - level=logging.DEBUG, - format="%(asctime)s %(message)s" -) +# setup command group +@click.group() +def cli(): + # print lumos header + header_ascii_art = text2art("Lumos", font="big") + click.echo(header_ascii_art) + + pass -# collect parameters -cellpaintingchannels = [x[1] for x in parameters.cellplainting_channels] -@click.command() +# ----------------------------- QUALITY CONTROL ----------------------------- # + + +# setup QC mode +@cli.command( + name="qc", + help="Quality Control", +) @click.option( + '-s', "--scope", type=click.Choice(["run", "plate", "channel"], case_sensitive=False), + required=True, help="If you want to generate a plateview from a single channel, plate or whole run", ) @click.option( + '-c', "--channel", - type=click.Choice(cellpaintingchannels, case_sensitive=True), - help="For single channel render only", + type=click.Choice(cellpainting_channels_list, case_sensitive=True), + help="For single channel render only.", ) @click.option( + '-sp', "--source-path", type=click.Path(exists=True), - help="Folder of your run or single plate", + required=True, + help="Folder of your run or single plate.", ) @click.option( + '-op', "--output-path", type=click.Path(exists=True), - help="Folder where images will be output", + required=True, + help="Folder where images will be outputted.", ) @click.option( + '-tp', "--temp-path", type=click.Path(exists=True), - help="Temporary working folder path. Default will be: " - + str(default_temp_directory), default=default_temp_directory, + show_default=True, + help="Path to the temporary working folder", +) +@click.option( + '-b', + "--brightfield", + type=click.Choice(["1", "2", "3", "all"], case_sensitive=False), + help="Choose which brightfield channels to render, none are rendered by default.", +) +@click.option( + '-p', + "--parallelism", + type=click.INT, + default=1, + show_default=True, + help="Choose the number of CPU cores on which to perform parallel computation of different channels.", +) +@click.option( + '-k', + "--keep-temp-files", + is_flag=True, + help="(dev) Choose if temporary files should be kept and not overwriten by a new copy of the source files.", ) -def start(scope, channel, source_path, output_path, temp_path): - """Lumos CLI - Cell painting plate image generator - """ +def quality_control(scope, channel, source_path, output_path, temp_path, brightfield, parallelism, keep_temp_files): - # annpunce startup - logging.info("Started") + is_in_parallel = (parallelism != 1) - # Check conditions to process run, plate or single channel - if not scope: - click.echo("Please define a scope") - return + # create logger + logger.setup(temp_path, is_in_parallel) - if scope == "run": + # announce startup + logger.info("Started - Quality Control") + + # decode arguments + if is_in_parallel: + if parallelism < 1: + click.echo("CLI ERROR: '--paralellism' argument cannot be less than 1. Please remove it or change its value.") + return + click.echo("CLI WARNING: When using parallelism, Keyboard Interrupts (CTRL+C) do not terminate the program.") + click.echo(" To halt the execution of the program before it finishes normally, you have to close your terminal.") + click.echo() + if keep_temp_files: + click.echo("CLI Note: Keeping previously downloaded temporary files (remove CLI argument '-k' or '--keep-temp-files' to regenerate them every time)") + logger.info("Argument '--keep-temp-files' used from CLI: keeping previously downloaded temporary files") + + if scope != "channel": if channel: - click.echo( - "--channel argument must not be used for run generation. please remove it" - ) + click.echo("CLI ERROR: '--channel' argument must not be used for run/plate generation. Please remove it.") return + channels_to_render = parameters.default_channels_to_render.copy() + if brightfield == "1": + channels_to_render.append("Z01C06") + click.echo("CLI Note: Generating render ONLY for brightfield channel Z01C06.") + elif brightfield == "2": + channels_to_render.append("Z02C06") + click.echo("CLI Note: Generating render ONLY for brightfield channel Z02C06.") + elif brightfield == "3": + channels_to_render.append("Z03C06") + click.echo("CLI Note: Generating render ONLY for brightfield channel Z03C06.") + elif brightfield == "all": + channels_to_render.append("Z01C06") + channels_to_render.append("Z02C06") + channels_to_render.append("Z03C06") + click.echo("CLI Note: Generating renders for ALL brightfield channels.") + else: + click.echo("CLI Note: Generating NO render for brightfield channels.") + click.echo(os.linesep) + + # execute image generation according to the scope + if scope == "run": + # get run name run_name = Path(source_path) # get plates and their path, only if files in it - click.echo("Scan to detect plate folders..") - run_folder_list = Path(source_path).glob("**") # create a dict with plate name as key and plate folder path as value # only folders with tif images are eligible - folder_list = { + source_folder_dict = { x.name: x for x in run_folder_list if (x.is_dir() and len(fnmatch.filter(os.listdir(x), "*.tif"))) @@ -100,83 +169,174 @@ def start(scope, channel, source_path, output_path, temp_path): click.echo( "Lumos will process " - + str(len(folder_list)) - + " Folders from run:" + + str(len(source_folder_dict)) + + " plate folders from run: " + str(run_name) ) - click.echo("PLates: " + str(list(folder_list.keys()))) + click.echo("Plates: " + str(list(source_folder_dict.keys()))) + click.echo( + "Channels being rendered: " + + str(channels_to_render) + + os.linesep + + os.linesep + ) - # generate # render all the plates of the run render_single_run_plateview( - source_path, - folder_list, - parameters.default_per_plate_channel_to_render, - parameters.cellplainting_channels_dict, + source_folder_dict, + channels_to_render, output_path, temp_path, + parallelism, + keep_temp_files, ) elif scope == "plate": - if channel: - click.echo( - "--channel argument must not be used for plate generation. please remove it" - ) - return - # get platename plate_name = Path(source_path).name click.echo( "Process plate: " + plate_name + " and render channels: " - + str(parameters.default_per_plate_channel_to_render), + + str(channels_to_render), ) # render all the channels of the plate - render_single_plate_plateview( + if not is_in_parallel: + render_single_plate_plateview( + source_path, + plate_name, + channels_to_render, + output_path, + temp_path, + keep_temp_files, + ) + else: + render_single_plate_plateview_parallelism( + source_path, + plate_name, + channels_to_render, + output_path, + temp_path, + parallelism, + keep_temp_files, + ) + + elif scope == "channel": + + if not channel: + click.echo("Missing channel, please define a channel") + return + + # get platename + plate_name = Path(source_path).name + click.echo( + "Process plate: " + + plate_name, + ) + click.echo( + "Render channel: " + + channel + + " - " + + str(parameters.cellplainting_channels_dict[channel]), + ) + + # render the channel for the plate + render_single_channel_plateview( source_path, plate_name, - parameters.default_per_plate_channel_to_render, - parameters.cellplainting_channels_dict, + channel, + parameters.cellplainting_channels_dict[channel], output_path, temp_path, + keep_temp_files, ) - elif scope == "channel": + # announce stop + logger.info("Stopped - Quality Control") + + return - if not channel: - click.echo("Missing channel, please define a channel") - return - if not source_path: - click.echo("Missing source plate path, please define a path") - else: - # get platename - plate_name = Path(source_path).name - click.echo( - "Process plate:" - + plate_name - + " channel:" - + channel - + " " - + str(parameters.cellplainting_channels_dict[channel]), - ) +# ----------------------------- CELL PAINTING ----------------------------- # - # render the channel for the plate - render_single_channel_plateview( - source_path, - plate_name, - channel, - parameters.cellplainting_channels_dict[channel], - output_path, - temp_path, - ) - # annpunce stop - logging.info("Stop") +# define cellpainter command +@cli.command( + name="cp", + help="Cell Painting", +) +@click.option( + "--style", + type=click.Choice(fingerprint_style_list, case_sensitive=False), + default=fingerprint_style_list[0], + show_default=True, + help="Choose the rendering style of the output image.", +) +@click.option( + '-s', + "--scope", + type=click.Choice(["plate", "wells"], case_sensitive=False), + required=True, + help="If you want to generate a cellpainted image for a whole plate, or each individual well.", +) +@click.option( + '-sp', + "--source-path", + type=click.Path(exists=True), + required=True, + help="Folder of your run or single plate.", +) +@click.option( + '-op', + "--output-path", + type=click.Path(exists=True), + required=True, + help="Folder where images will be outputted.", +) +@click.option( + '-tp', + "--temp-path", + type=click.Path(exists=True), + default=default_temp_directory, + show_default=True, + help="Path to the temporary working folder", +) +def cell_painting(scope, source_path, output_path, temp_path, style): + + # create logger + logger.setup(temp_path, False) + + # announce startup + logger.info("Started - Cell Painting") + + # get platename + plate_name = Path(source_path).name + + click.echo( + "Process wells of plate: " + + plate_name + + " and multiplex cell painting channels C01,C02,C03,C04,C05" + ) + + # multiplex the channels of the plate (not brightfield) into a single RGB image + picasso_generate_plate_image( + source_path, + plate_name, + output_path, + temp_path, + style, + scope, + True, + ) + + # announce stop + logger.info("Stopped - Cell Painting") + + return if __name__ == "__main__": - start() + # use Click + cli() diff --git a/readme-dev.md b/readme-dev.md new file mode 100644 index 0000000..acf99a3 --- /dev/null +++ b/readme-dev.md @@ -0,0 +1,24 @@ +## Developing the Lumos package + +### Local installation + +During development, the command ```$ python setup.py develop``` can be used to install the package locally and therefore test the `lumos` command. After development, the command ```$ python setup.py develop --uninstall``` can be used to reverse the operation. + +*Note: this is compatible with 'hot-reload', i.e. the `lumos` command will always use the latest version of the source files.* + +### Testing + +To test your current version of Lumos, install the package (e.g. locally with the command above) and run ```$ pytest -v``` in the root folder of the program. +When creating tests, if print statements are included inside of them for debugging purposes, use ```$ pytest -s``` instead to print them to the console. + +### Generating the HTML documentation + +The HTML documentation that is provided alongside the source code is generated using the `pdoc3` package. + +This package uses the XML documentation present in the source files to generate a static HTML website. + +Once the XML documentation is well completed, use the command `$ pdoc3 --html ./lumos` to generate the HTML documentation of the sub-modules contained in the `./lumos` subfolder, i.e. the back-end modules (*for some reason, running this command for the root folder of the codebase `.` gives an error, so only the documentation for the back-end modules can be generated*). + +*Note: if some HTML documentation has already been recently generated, use the `--force` flag to overwrite it.* + +Once the HTML files have been successfully generated, they can be placed inside of the `./docs` folder. \ No newline at end of file diff --git a/readme.md b/readme.md index df637a4..e07ecc6 100644 --- a/readme.md +++ b/readme.md @@ -1,23 +1,103 @@ # Lumos -Lumos is a first version of a python script to aggregate pictures obtained from cellpainting assay. This version of lumos fits with images generated with Yokogawa CV8000 High content analyis system. +Lumos is a script to generate full-plate images from separate well pictures obtained from the cellpainting assay, as well as to generate cell-painted images. -Your images must be accessible from your filesystem. +Important notes: +- This version of Lumos fits with images generated with the Yokogawa CV8000 High content analysis system. +- Your images must be accessible from your file system. -## Setup -Create a virtualenv & activate it +
- python -m venv venv - source venv/bin/activate +In this document, you will find information related to: + * [The installation of the package](#installation-from-the-source-files) + * [How to use the program, i.e. the Command-line interface](#command-line-interface-cli) -Install depenencies & Install lumos +
- pip install -r requirements.txt - python setup.py install +To find instructions for developers or maintainers of the project, please refer to [this other documentation](./readme-dev.md). +
-## Use +## Installation from the source files -From your prompt, launch help to see the arguments required. +### Prerequisites - lumos --help +You need to have Python 3.8 installed on a Windows or Linux machine. + +### Get the source files + +- Download the source files, either by downloading the ZIP file, or cloning the repository. + +- Extract/copy them in a 'lumos' folder somewhere on your machine. + +### Install Lumos and its dependencies + +- Go with a PowerShell terminal in the 'lumos' folder. + +- Create a Python VirtualEnv: ```$ python -m venv venv``` + +- Activate the VirtualEnv: ```$ ./venv/Script/activate.ps1``` + +- Install the dependencies using PIP: ```$ pip install -r .\requirements.txt``` + +- Install Lumos: ```$ python setup.py install``` + +### Run Lumos + +- Make sure your VirtualEnv is activated. + +- Type in the console ```lumos``` to launch the program, or ```lumos --help``` for instructions on how to use it. + +
+ +## Command-line Interface (CLI) + +The Lumos program is entirely controlled from the terminal. Once installed, you will be able to start the program with the ```lumos``` command, followed by an operation mode, and some arguments. + +To see the most up-to-date documentation on the CLI, always run the ```$ lumos --help``` command, and ```$ lumos --help``` for more detailed documentation on a specific command/argument. + +However, we will provide in this document a more detailed and comprehensive explanation of how to use the CLI of Lumos. + +### Operation modes + +Lumos has currently two modes of operation: + * [Quality Control (`qc`)](#quality-control) + * [Cell Painting (`cp`)](#cell-painting) + +To choose which mode the Lumos program should use, specify its identifier after the `lumos` keyword as follows: + - Use `$ lumos qc ` to use the Quality Control mode. + - Use `$ lumos cp ` to use the Cell Painting mode. + +Here is a brief description of each mode: + +> #### Quality Control: +> +> This mode will assemble all separate pictures of a plate's sites for a specific channel into one single greyscale image. This can be done for a single channel, every channel of a plate, or even several plates in a row. +> +> This is used to quickly visualize any imperfections or problems that a plate might have exhibited during the experiment. + +> #### Cell Painting: +> +> This mode will combine all separate pictures of a plate's sites, for all channels, into a single color image. The blending algorithm used for merging the independent channels together can be chosen, and an "accurate" style is given to try to reflect the emission wavelengths associated with each channel. +> +> This can also be used to some extent for some Quality Control, but is mainly intended for the production of resources/assets for communications about the project. + +### Arguments + +Each operation mode requires arguments to function correctly. Some of them are **mandatory** (such as where the source images can be found), and some are **optional** (such as if any Brightfield channels should be rendered). + +Please find bellow the table of all arguments that can be used in Lumos: + +| Operation mode | Argument name | Alias | Required | Expected value | Description | +|----------------|-----------------|-------|--------------------------|------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Both | `--scope` | `-s` | Yes | One of [channel, plate, run] if in `qc`, or one of [plate, wells] if in `cp` | The scope indicates to the program what should be rendered and outputted. | +| qc | `--channel` | `-c` | Yes, if scope is channel | One of the channels' name (run `$ lumos qc --help` for full list) | When the scope is "channel" in QC mode, this argument is required in order to indicate which channel should be rendered. | +| Both | `--source-path` | `-sp` | Yes | A valid path | The path to the source site images. | +| Both | `--output-path` | `-op` | Yes | A valid path | The path where the output images should be stored. | +| Both | `--temp-path` | `-tp` | No | A valid path, if present | The path where temporary files should be stored during the operation of the program. If omitted, the default is the system's temp directory. | +| qc | `--brightfield` | `-b` | No | One of [1,2,3,all], if present | Chooses which brightfield channels to render. If omitted, none will be rendered. | +| qc | `--parallelism` | `-p` | No | An integer number, if present | Chooses on how many CPU cores to split the load of the program. If omitted, this will be 1. If present, less information will be printed to the console during the operation of the program. | +| cp | `--style` | | Yes | One of the available styles' name (run `$ lumos cp --help` for full list) | Chooses which rendering style to use when generating a cell-painted image. | + + +*Note: when running `$ lumos qc --help`, you might see arguments that are not included in the table above, such as `--keep-temp-files`. These arguments are for development and debugging purposes only, and should not be used in a normal operation of Lumos.* diff --git a/setup.py b/setup.py index 2eee9c9..16bcb06 100644 --- a/setup.py +++ b/setup.py @@ -4,26 +4,27 @@ long_description = fh.read() with open("requirements.txt", "r", encoding="utf-8") as fh: requirements = fh.read() + setup( - name = 'lumos', - version = '0.0.2', - author = 'DSDM', - author_email = '', - license = 'NOLICENCE', - description = 'A cell painting plateviewer', - long_description = long_description, - long_description_content_type = "text/markdown", - url = 'https://github.com/nicolasboisseau/lumos', - py_modules = ['lumoscli','lumos'], - packages = find_packages(), - install_requires = [requirements], + name='lumos', + version='0.0.3', + author='DSDM - Institut De Recherches Servier', + author_email='nicolas.boisseau@servier.com', + license='NOLICENCE', + description='A cell painting plateviewer', + long_description=long_description, + long_description_content_type="text/markdown", + url='https://github.com/servier-github/jumpcp-aws', + py_modules=['lumoscli', 'lumos'], + packages=find_packages(), + install_requires=[requirements], python_requires='>=3.8', classifiers=[ "Programming Language :: Python :: 3.8", "Operating System :: OS Independent", ], - entry_points = ''' + entry_points=''' [console_scripts] - lumos=lumoscli:start + lumos=lumoscli:cli ''' -) \ No newline at end of file +) diff --git a/tests/test_load_image.py b/tests/test_1_toolbox_load_image.py similarity index 58% rename from tests/test_load_image.py rename to tests/test_1_toolbox_load_image.py index 670dd52..59e219f 100644 --- a/tests/test_load_image.py +++ b/tests/test_1_toolbox_load_image.py @@ -5,16 +5,19 @@ import pandas as pd from pathlib import Path import os + from lumos.toolbox import load_site_image + # Arrange @pytest.fixture def fake_image(): """ Generate a fake image and save it in temp folder """ img = np.full((1000, 1000, 1), 32768, np.uint16) - - fake_16_bit_image_file = tempfile.NamedTemporaryFile(delete=False,suffix='.tif') + + fake_16_bit_image_file = tempfile.NamedTemporaryFile( + delete=False, suffix='.tif') cv2.imwrite( fake_16_bit_image_file.name, @@ -23,22 +26,26 @@ def fake_image(): return fake_16_bit_image_file.name + @pytest.fixture def fake_dataframe(fake_image): """ generate a dataframe with the following columns ["well", "site", "filename", "fullpath"] - """ image_path = Path(fake_image) - name = image_path.name - parent = image_path.parent - dict = [{"well":"A01", "site":1, "filename":image_path.name, "fullpath":image_path.parent}] - fake_df = pd.DataFrame.from_dict(dict,orient='columns') + dict = [{"well": "A01", "site": 1, "filename": image_path.name, + "fullpath": image_path.parent}] + fake_df = pd.DataFrame.from_dict(dict, orient='columns') return fake_df -def test_load_image(fake_dataframe,fake_image): + +def test_load_image(fake_dataframe, fake_image): # Act fake_image_path = Path(fake_image) - site_image = load_site_image(1,fake_dataframe,str(fake_image_path.parent)) + site_image = load_site_image( + 1, fake_dataframe, str(fake_image_path.parent)) # Assert - assert site_image.shape == (1000,1000) \ No newline at end of file + assert site_image.shape == (1000, 1000) + + # Clean-up + os.remove(fake_image_path) diff --git a/tests/test_2_qc_pipeline.py b/tests/test_2_qc_pipeline.py new file mode 100644 index 0000000..5345073 --- /dev/null +++ b/tests/test_2_qc_pipeline.py @@ -0,0 +1,99 @@ +import pytest +import cv2 +import tempfile +import numpy as np +import os + +import lumos.parameters +import lumos.toolbox +import lumos.generator + + +# Arrange +@pytest.fixture +def fake_placeholder(): + placeholder_img = np.full( + shape=(int(1000), int(1000), 1), + fill_value=lumos.parameters.placeholder_background_intensity, + dtype=np.uint8 + ) + placeholder_img = lumos.toolbox.draw_markers( + placeholder_img, lumos.parameters.placeholder_markers_intensity) + + return placeholder_img + + +def test_qc_pipeline(fake_placeholder): + # Act + fill_value = 65535 + img = np.full((1000, 1000, 1), fill_value, np.uint16) + source_folder = tempfile.TemporaryDirectory() + # save the fake images in the temp folder, one for each channel + cv2.imwrite( + source_folder.name+"/DestTestQC_A01_T0001F001L01A01Z01C01.tif", + img, + ) + cv2.imwrite( + source_folder.name+"/DestTestQC_A05_T0001F002L01A01Z01C02.tif", + img, + ) + cv2.imwrite( + source_folder.name+"/DestTestQC_B21_T0001F003L01A01Z01C03.tif", + img, + ) + cv2.imwrite( + source_folder.name+"/DestTestQC_I12_T0001F005L01A01Z01C04.tif", + img, + ) + cv2.imwrite( + source_folder.name+"/DestTestQC_P24_T0001F006L01A01Z01C05.tif", + img, + ) + + output_folder = tempfile.TemporaryDirectory() + temporary_folder = tempfile.TemporaryDirectory() + plate_name = "DestTestQC" + + lumos.generator.render_single_plate_plateview( + source_folder.name, + plate_name, + lumos.parameters.default_channels_to_render, + output_folder.name, + temporary_folder.name, + False, + ) + + # Assert + for channel_to_test in lumos.parameters.default_channels_to_render: + + output_channel_image_path = ( + output_folder.name + + "/" + + plate_name + + "-" + + channel_to_test + + "-" + + str(lumos.parameters.channel_coefficients[channel_to_test]) + + ".jpg" + ) + + # Test that there is an output for the channel + assert(os.path.isfile(output_channel_image_path)) + + output_channel_image = cv2.imread(output_channel_image_path) + + # Test that the output has the expected shape + expected_width = int(3000 * 24 * lumos.parameters.rescale_ratio) + expected_height = int(2000 * 16 * lumos.parameters.rescale_ratio) + assert(output_channel_image.shape == (expected_height, expected_width, 3)) + + # Test that the output has around the expected intensity (with margin because of well labels) + expected_mean = (2303 * np.mean(fake_placeholder) + + (np.mean(img)/256)) / 2304 + assert(abs(expected_mean - np.mean(output_channel_image)) <= 0.5) + + # Uncomment the following line to save the generated test outputs: + # cv2.imwrite(tempfile.gettempdir()+"/DestTestQC_output_" + # + channel_to_test+".tif", output_channel_image) + + return diff --git a/tests/test_3_cp_pipeline.py b/tests/test_3_cp_pipeline.py new file mode 100644 index 0000000..d113a74 --- /dev/null +++ b/tests/test_3_cp_pipeline.py @@ -0,0 +1,78 @@ +import pytest +import cv2 +import tempfile +import numpy as np +import os + +import lumos.parameters +import lumos.toolbox +import lumos.picasso + + +def test_cp_pipeline(): + # Act + fill_value = 65535 + img = np.full((1000, 1000, 1), fill_value, np.uint16) + source_folder = tempfile.TemporaryDirectory() + # save the fake images in the temp folder, one for each channel + cv2.imwrite( + source_folder.name+"/DestTestCP_A01_T0001F001L01A01Z01C01.tif", + img, + ) + cv2.imwrite( + source_folder.name+"/DestTestCP_A05_T0001F002L01A01Z01C02.tif", + img, + ) + cv2.imwrite( + source_folder.name+"/DestTestCP_B21_T0001F003L01A01Z01C03.tif", + img, + ) + cv2.imwrite( + source_folder.name+"/DestTestCP_I12_T0001F005L01A01Z01C04.tif", + img, + ) + cv2.imwrite( + source_folder.name+"/DestTestCP_P24_T0001F006L01A01Z01C05.tif", + img, + ) + + output_folder = tempfile.TemporaryDirectory() + temporary_folder = tempfile.TemporaryDirectory() + plate_name = "DestTestCP" + style = "accurate" + + lumos.picasso.picasso_generate_plate_image( + source_folder.name, + plate_name, + output_folder.name, + temporary_folder.name, + style, + scope='plate', + display_well_details=False, + ) + + # Assert + output_image_path = ( + output_folder.name + + "/" + + plate_name + + "-picasso-" + + style + + ".jpg" + ) + + # Test that there is an output + assert(os.path.isfile(output_image_path)) + + output_image = cv2.imread(output_image_path) + + # Test that the output has the expected shape + expected_width = int(3000 * 24 * lumos.parameters.rescale_ratio_picasso_plate) + expected_height = int(2000 * 16 * lumos.parameters.rescale_ratio_picasso_plate) + assert(output_image.shape == (expected_height, expected_width, 3)) + + # Uncomment the following line to save the generated test output: + # cv2.imwrite(tempfile.gettempdir()+"/DestTestCP_output_" + + # style+".tif", output_image) + + return