Merge pull request #10 from sreekarreddydfci/dev-tilestitching

Merging changes from dev_graph

CONTRIBUTING.rst

@@ -59,18 +59,23 @@ Setting up a local development environment
 Running tests
 -------------
 
-To run the full testing suite:
+To run the full testing suite (not recommended):
 
 .. code-block::
 
-    python -m pytest
+    python -m pytest 
 
-Some tests are known to be very slow. To skip them, run instead:
+Some tests are known to be very slow. Tests for the tile stitching functionality must be ran separately. To skip them, run:
 
 .. code-block::
 
-    python -m pytest -m "not slow"
+    python -m pytest -m "not slow and not exclude"
 
+Then, run the tilestitching test:
+
+.. code-block::
+
+    python -m pytest tests/preprocessing_tests/test_tilestitcher.py
 
 Building documentation locally
 ------------------------------
@@ -89,7 +94,9 @@ Checking code coverage
 .. code-block::
 
     conda install coverage  # install coverage package for code coverage
-    coverage run            # run tests and calculate code coverage
+    COVERAGE_FILE=.coverage_others coverage run -m pytest -m "not slow and not exclude" # run coverage for all files except tile stitching
+    COVERAGE_FILE=.coverage_tilestitcher coverage run -m pytest tests/preprocessing_tests/test_tilestitcher.py # run coverage for tile stitching
+    coverage combine .coverage_tilestitcher .coverage_others # combine coverage results
     coverage report         # view coverage report
     coverage html           # optionally generate HTML coverage report
 
@@ -163,9 +170,10 @@ To run the test suite and check code coverage:
 
 .. code-block::
 
-    conda install pytest    # first install pytest package
     conda install coverage  # install coverage package for code coverage
-    coverage run            # run tests and calculate code coverage
+    COVERAGE_FILE=.coverage_others coverage run -m pytest -m "not slow and not exclude" # run coverage for all files except tile stitching
+    COVERAGE_FILE=.coverage_tilestitcher coverage run -m pytest tests/preprocessing_tests/test_tilestitcher.py # run coverage for tile stitching
+    coverage combine .coverage_tilestitcher .coverage_others # combine coverage results
     coverage report         # view coverage report
     coverage html           # optionally generate HTML coverage report
 

README.md

@@ -38,9 +38,9 @@ We recommend using [Conda](https://conda.io/projects/conda/en/latest/user-guide/
 
 #### Installing Conda
 
-If you don't have Conda installed, you can download Miniconda [here]. (https://docs.conda.io/en/latest/miniconda.html)
+If you don't have Conda installed, you can download Miniconda [here](https://docs.conda.io/en/latest/miniconda.html).
 
-#### Updating Conda and Using libmamba
+#### Updating Conda and Using libmamba (Optional)
 
 Recent versions of Conda have integrated `libmamba`, a faster dependency solver. To benefit from this improvement, first ensure your Conda is updated:
 
@@ -54,23 +54,12 @@ Then, to install and set the new `libmamba` solver, run:
 conda install -n base conda-libmamba-solver
 conda config --set solver libmamba
 ````
-
 *Note: these instructions are for Linux. Commands may be different for other platforms.*
 
-## 2. PathML Installation Methods
-
-### 2.1 Install with pip (Recommended for Users)
-
-#### Common Steps
-
-Create and Activate Conda Environment:
-````
-conda create --name pathml python=3.9
-conda activate pathml
-````
-
 #### Platform-Specific External Dependencies
 
+For installation methods [1)](#2.1-Install-with-pip-(Recommended-for-Users)) and [2)](#2.2-Install-from-Source-(Recommended-for-Developers)), you will need to install the following platform-specific packages. 
+
 * Linux: Install external dependencies with [Apt](https://ubuntu.com/server/docs/package-management):
 ````
 sudo apt-get install openslide-tools g++ gcc libblas-dev liblapack-dev
@@ -94,7 +83,17 @@ For Windows users, an alternative to using `vcpkg` is to download and use pre-bu
   - Download the OpenSlide Windows binaries from the [OpenSlide Downloads](https://openslide.org/download/) page.
   - Extract the archive to your desired location, e.g., `C:\OpenSlide\`.
 
-#### Install OpenJDK 17
+
+## 2. PathML Installation Methods
+
+### 2.1 Install with pip (Recommended for Users)
+
+#### Create and Activate Conda Environment
+````
+conda create --name pathml python=3.9
+conda activate pathml
+````
+#### Install OpenJDK 
 ````
 conda install -c conda-forge 'openjdk<=18.0'
 ````
@@ -106,21 +105,30 @@ pip install pathml
 
 ### 2.2 Install from Source (Recommended for Developers)
 
-Clone repo:
+#### Clone repository
 ````
 git clone https://github.com/Dana-Farber-AIOS/pathml.git
 cd pathml
 ````
 
-Create conda environment:
+#### Create conda environment 
+
+* Linux and Windows:
 
 ````
 conda env create -f environment.yml
 conda activate pathml
 ````
-To use GPU acceleration for model training or other tasks, you must install CUDA. The default CUDA version in our environment file is 11.6. To install a different CUDA version, refer to the instructions [here](##CUDA)). 
+To use GPU acceleration for model training or other tasks, you must install CUDA. The default CUDA version in our environment file is 11.6. To install a different CUDA version, refer to the instructions [here](#CUDA)). 
+
+* MacOS:
+
+````
+conda env create -f requirements/environment_mac.yml
+conda activate pathml
+````
 
-Install `PathML` from source: 
+#### Install `PathML` from source: 
 ````
 pip install -e .
 ````

docs/source/api_inference_reference.rst

@@ -21,6 +21,11 @@ RemoteTestHoverNet Class
 
 .. autoapiclass:: pathml.inference.RemoteTestHoverNet
 
+RemoteMesmer Class
+------------------------
+
+.. autoapiclass:: pathml.inference.RemoteMesmer
+
 Helper functions
 ^^^^^^^^^^^^^^^^
 

examples/InferenceOnnx_tutorial.ipynb

@@ -240,7 +240,24 @@
     "                 'model_input_notes': 'Accepts tiles of 256 x 256',\n",
     "                 'model_output_notes': None,\n",
     "                 'citation': 'Pocock J, Graham S, Vu QD, Jahanifar M, Deshpande S, Hadjigeorghiou G, Shephard A, Bashir RM, Bilal M, Lu W, Epstein D. TIAToolbox as an end-to-end library for advanced tissue image analytics. Communications medicine. 2022 Sep 24;2(1):120.'}\n",
-    "             ```"
+    "             ```\n",
+    "\n",
+    "<br> \n",
+    "\n",
+    "- `RemoteMesmer` \n",
+    "    - This class inherits from `Inference` and is hosted on `HuggingFace`. \n",
+    "    - `local` is automatically set to `False` \n",
+    "    - This model is from [Deepcell](https://github.com/vanvalenlab/deepcell-tf/blob/master/deepcell/applications/mesmer.py)\n",
+    "    - Greenwald NF, Miller G, Moen E, Kong A, Kagel A, Dougherty T, Fullaway CC, McIntosh BJ, Leow KX, Schwartz MS, Pavelchek C. Whole-cell segmentation of tissue images with human-level performance using large-scale data annotation and deep learning. Nature biotechnology. 2022 Apr;40(4):555-65.\n",
+    "    - Its `model_card` is:\n",
+    "        -   ```python \n",
+    "                {'name': \"Deepcell's Mesmer\",\n",
+    "                 'num_classes': 3,\n",
+    "                 'model_type': 'Segmentation',\n",
+    "                 'notes': None,\n",
+    "                 'model_input_notes': 'Accepts tiles of 256 x 256',\n",
+    "                 'model_output_notes': None,\n",
+    "                 'citation': 'Greenwald NF, Miller G, Moen E, Kong A, Kagel A, Dougherty T, Fullaway CC, McIntosh BJ, Leow KX, Schwartz MS, Pavelchek C. Whole-cell segmentation of tissue images with human-level performance using large-scale data annotation and deep learning. Nature biotechnology. 2022 Apr;40(4):555-65.'}"
    ]
   },
   {

pathml/inference/inference.py

@@ -442,17 +442,17 @@ class RemoteMesmer(Inference):
     Nature biotechnology. 2022 Apr;40(4):555-65.
 
     Args:
-    model_path (str): temp file name to download onnx from huggingface, do not change
-    input_name (str): name of the input the ONNX model accepts, default = "data", do not change
-    num_classes (int): number of classes you are predicting, do not change
-    model_type (str): type of model, e.g. "segmentation", do not change
-    local (bool): True if the model is stored locally, default = "True", do not change
-    nuclear_channel(int): channel that defines cell nucleus
-    cytoplasm_channel(int): channel that defines cell membrane or cytoplasm
-    image_resolution(float): pixel resolution of image in microns. Currently only supports 0.5
-    preprocess_kwargs(dict): keyword arguemnts to pass to pre-processing function
-    postprocess_kwargs_nuclear(dict): keyword arguments to pass to post-processing function
-    postprocess_kwargs_whole_cell(dict): keyword arguments to pass to post-processing function
+        model_path (str): temp file name to download onnx from huggingface, do not change
+        input_name (str): name of the input the ONNX model accepts, default = "data", do not change
+        num_classes (int): number of classes you are predicting, do not change
+        model_type (str): type of model, e.g. "segmentation", do not change
+        local (bool): True if the model is stored locally, default = "True", do not change
+        nuclear_channel(int): channel that defines cell nucleus
+        cytoplasm_channel(int): channel that defines cell membrane or cytoplasm
+        image_resolution(float): pixel resolution of image in microns. Currently only supports 0.5
+        preprocess_kwargs(dict): keyword arguemnts to pass to pre-processing function
+        postprocess_kwargs_nuclear(dict): keyword arguments to pass to post-processing function
+        postprocess_kwargs_whole_cell(dict): keyword arguments to pass to post-processing function
     """
 
     def __init__(

pathml/preprocessing/tilestitcher.py

@@ -123,7 +123,11 @@ def _import_qupath_classes(self):
     def format_jvm_options(qupath_jars, memory):
         memory_option = f"-Xmx{memory}"
         formatted_classpath = [
-            path.replace("/", "\\") if platform.system() == "Windows" else path
+            (
+                str(Path(path).as_posix())
+                if platform.system() != "Windows"
+                else str(Path(path))
+            )
             for path in qupath_jars
         ]
         class_path_option = "-Djava.class.path=" + os.pathsep.join(formatted_classpath)
@@ -171,6 +175,7 @@ def setup_bfconvert(self, bfconvert_dir):
 
         try:
             if not tools_dir.exists():
+
                 tools_dir.mkdir(parents=True, exist_ok=True)
 
             if not bftools_dir.exists():
@@ -204,6 +209,7 @@ def setup_bfconvert(self, bfconvert_dir):
             zipfile.BadZipFile,
             PermissionError,
             subprocess.CalledProcessError,
+            OSError,
         ) as e:
             raise BFConvertSetupError(f"Error setting up bfconvert: {e}")
 

requirements/environment_mac.yml

@@ -0,0 +1,37 @@
+name: pathml
+
+channels:
+    - conda-forge
+    - pytorch
+
+dependencies:
+    - python<=3.10
+    - pip==23.3.2
+    - numpy==1.23.5
+    - scipy<=1.11.4
+    - scikit-image<=0.22.0
+    - matplotlib<=3.8.2
+    - openjdk<=18.0.0
+    - h5py==3.10.0
+    - dask<=2023.12.1
+    - pydicom==2.4.4
+    - pytest==7.4.3 
+    - pre-commit<=3.6.0
+    - coverage==7.3.4
+    - networkx<=3.2.1
+    - pip:
+        - torch==1.13.1
+        - python-bioformats==4.0.7
+        - python-javabridge==4.0.3
+        - protobuf==3.20.3
+        - onnx==1.15.0
+        - onnxruntime==1.16.3
+        - opencv-contrib-python==4.8.1.78
+        - openslide-python==1.3.1
+        - scanpy==1.9.6
+        - anndata<=0.10.3
+        - tqdm==4.66.1
+        - loguru==0.7.2
+        - pandas<=2.1.4
+        - torch-geometric==2.3.1
+        - jpype1==1.4.1

requirements/environment_test.yml

@@ -24,7 +24,6 @@ dependencies:
         - python-bioformats==4.0.7
         - python-javabridge==4.0.3
         - protobuf==3.20.3
-        - deepcell<=0.12.7
         - onnx==1.15.0
         - onnxruntime==1.16.3
         - opencv-contrib-python==4.8.1.78

tests/preprocessing_tests/test_tilestitcher.py

@@ -1,5 +1,11 @@
+"""
+Copyright 2021, Dana-Farber Cancer Institute and Weill Cornell Medicine
+License: GNU GPL 2.0
+"""
+
 import glob
 import os
+import platform
 import subprocess
 import tempfile
 import urllib
@@ -122,7 +128,7 @@ def test_format_jvm_options_memory(memory, expected_memory_option, tile_stitcher
 
 @pytest.mark.exclude
 @pytest.mark.parametrize(
-    "qupath_jars, expected_classpath",
+    "qupath_jars, expected_classpath_suffix",
     [
         ([], ""),
         (
@@ -131,16 +137,22 @@ def test_format_jvm_options_memory(memory, expected_memory_option, tile_stitcher
         ),
         (
             ["C:\\path\\to\\jar1.jar", "C:\\path\\to\\jar2.jar"],
-            "C:\\path\\to\\jar1.jar;C:\\path\\to\\jar2.jar",
+            "C:\\path\\to\\jar1.jar;C:\\path\\to\\jar2.jar",  # Adjusted to use backslashes and semicolon for Windows
         ),
     ],
 )
 def test_format_jvm_options_classpath(
-    qupath_jars, expected_classpath, tile_stitcher, monkeypatch
+    qupath_jars, expected_classpath_suffix, tile_stitcher, monkeypatch
 ):
-    monkeypatch.setattr(os, "pathsep", ";" if os.name == "nt" else ":")
+    os_name = "nt" if any("C:\\" in jar for jar in qupath_jars) else "posix"
+    monkeypatch.setattr(
+        platform, "system", lambda: "Windows" if os_name == "nt" else "Linux"
+    )
+    monkeypatch.setattr(os, "pathsep", ";" if os_name == "nt" else ":")
+
     _, class_path_option = tile_stitcher.format_jvm_options(qupath_jars, "512m")
-    expected_classpath = "-Djava.class.path=" + os.pathsep.join(qupath_jars)
+    expected_classpath = "-Djava.class.path=" + expected_classpath_suffix
+
     assert class_path_option == expected_classpath
 
 
@@ -298,12 +310,6 @@ def test_bfconvert_version_output(tile_stitcher, bfconvert_setup, capsys):
     ), "bfconvert version not printed correctly"
 
 
-@pytest.mark.exclude
-def test_permission_error_on_directory_creation(tile_stitcher):
-    with pytest.raises(BFConvertSetupError):
-        tile_stitcher.setup_bfconvert("/fake/path")
-
-
 @pytest.mark.exclude
 @pytest.fixture
 def mock_subprocess(monkeypatch):
@@ -455,6 +461,8 @@ def test_run_bfconvert_no_delete_original(tile_stitcher, capsys):
 
     # Check if the original file still exists
     assert os.path.exists(stitched_image_path)
+    if os.path.exists(stitched_image_path):
+        os.unlink(stitched_image_path)
 
 
 @pytest.mark.exclude
@@ -613,16 +621,6 @@ def test_collect_tif_files_invalid_input(tile_stitcher):
     assert "Invalid input for collecting .tif files:" in str(exc_info.value)
 
 
-@pytest.mark.exclude
-@patch("os.chmod", side_effect=PermissionError("Permission denied"))
-def test_setup_bfconvert_permission_error(mock_chmod, tile_stitcher, bfconvert_dir):
-    with pytest.raises(BFConvertSetupError) as exc_info:
-        tile_stitcher.setup_bfconvert(bfconvert_dir)
-    assert "Permission error on setting executable flag: Permission denied" in str(
-        exc_info.value
-    )
-
-
 @pytest.mark.exclude
 def test_parse_region_missing_tags(tile_stitcher):