From 4306f80dbfc54e795a0540c8d1bb1bbf85462343 Mon Sep 17 00:00:00 2001 From: Varun Ullanat Date: Tue, 5 Mar 2024 17:55:44 +0000 Subject: [PATCH 1/8] Added environment for mac, updated instructions and docs --- CONTRIBUTING.rst | 22 ++++++++++----- README.md | 46 +++++++++++++++++++------------- requirements/environment_mac.yml | 37 +++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 26 deletions(-) create mode 100644 requirements/environment_mac.yml diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 0684c2df..49eecb6f 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -59,18 +59,23 @@ Setting up a local development environment Running tests ------------- -To run the full testing suite: +To run the full testing suite (not recommended): .. code-block:: - python -m pytest + python -m pytest -Some tests are known to be very slow. To skip them, run instead: +Some tests are known to be very slow. Tests for the tile stitching functionality must be ran separately. To skip them, run: .. code-block:: - python -m pytest -m "not slow" + python -m pytest -m "not slow and not exclude" +Then, run the tilestitching test: + +.. code-block:: + + python -m pytest tests/preprocessing_tests/test_tilestitcher.py Building documentation locally ------------------------------ @@ -89,7 +94,9 @@ Checking code coverage .. code-block:: conda install coverage # install coverage package for code coverage - coverage run # run tests and calculate code coverage + COVERAGE_FILE=.coverage_others coverage run -m pytest -m "not slow and not exclude" # run coverage for all files except tile stitching + COVERAGE_FILE=.coverage_tilestitcher coverage run -m pytest tests/preprocessing_tests/test_tilestitcher.py # run coverage for tile stitching + coverage combine .coverage_tilestitcher .coverage_others # combine coverage results coverage report # view coverage report coverage html # optionally generate HTML coverage report @@ -163,9 +170,10 @@ To run the test suite and check code coverage: .. code-block:: - conda install pytest # first install pytest package conda install coverage # install coverage package for code coverage - coverage run # run tests and calculate code coverage + COVERAGE_FILE=.coverage_others coverage run -m pytest -m "not slow and not exclude" # run coverage for all files except tile stitching + COVERAGE_FILE=.coverage_tilestitcher coverage run -m pytest tests/preprocessing_tests/test_tilestitcher.py # run coverage for tile stitching + coverage combine .coverage_tilestitcher .coverage_others # combine coverage results coverage report # view coverage report coverage html # optionally generate HTML coverage report diff --git a/README.md b/README.md index e7a542bd..ca419bb7 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ We recommend using [Conda](https://conda.io/projects/conda/en/latest/user-guide/ If you don't have Conda installed, you can download Miniconda [here]. (https://docs.conda.io/en/latest/miniconda.html) -#### Updating Conda and Using libmamba +#### Updating Conda and Using libmamba (Optional) Recent versions of Conda have integrated `libmamba`, a faster dependency solver. To benefit from this improvement, first ensure your Conda is updated: @@ -54,23 +54,12 @@ Then, to install and set the new `libmamba` solver, run: conda install -n base conda-libmamba-solver conda config --set solver libmamba ```` - *Note: these instructions are for Linux. Commands may be different for other platforms.* -## 2. PathML Installation Methods - -### 2.1 Install with pip (Recommended for Users) - -#### Common Steps - -Create and Activate Conda Environment: -```` -conda create --name pathml python=3.9 -conda activate pathml -```` - #### Platform-Specific External Dependencies +For installation methods [1)](#2.1-Install-with-pip-(Recommended-for-Users)) and [2)](#2.2-Install-from-Source-(Recommended-for-Developers)), you will need to install the following platform-specific packages. + * Linux: Install external dependencies with [Apt](https://ubuntu.com/server/docs/package-management): ```` sudo apt-get install openslide-tools g++ gcc libblas-dev liblapack-dev @@ -94,7 +83,17 @@ For Windows users, an alternative to using `vcpkg` is to download and use pre-bu - Download the OpenSlide Windows binaries from the [OpenSlide Downloads](https://openslide.org/download/) page. - Extract the archive to your desired location, e.g., `C:\OpenSlide\`. -#### Install OpenJDK 17 + +## 2. PathML Installation Methods + +### 2.1 Install with pip (Recommended for Users) + +#### Create and Activate Conda Environment +```` +conda create --name pathml python=3.9 +conda activate pathml +```` +#### Install OpenJDK ```` conda install -c conda-forge 'openjdk<=18.0' ```` @@ -106,21 +105,30 @@ pip install pathml ### 2.2 Install from Source (Recommended for Developers) -Clone repo: +#### Clone repository ```` git clone https://github.com/Dana-Farber-AIOS/pathml.git cd pathml ```` -Create conda environment: +#### Create conda environment + +* Linux and Windows: ```` conda env create -f environment.yml conda activate pathml ```` -To use GPU acceleration for model training or other tasks, you must install CUDA. The default CUDA version in our environment file is 11.6. To install a different CUDA version, refer to the instructions [here](##CUDA)). +To use GPU acceleration for model training or other tasks, you must install CUDA. The default CUDA version in our environment file is 11.6. To install a different CUDA version, refer to the instructions [here](#CUDA)). + +* MacOS: + +```` +conda env create -f requirements/environment_mac.yml +conda activate pathml +```` -Install `PathML` from source: +#### Install `PathML` from source: ```` pip install -e . ```` diff --git a/requirements/environment_mac.yml b/requirements/environment_mac.yml new file mode 100644 index 00000000..c2e6a390 --- /dev/null +++ b/requirements/environment_mac.yml @@ -0,0 +1,37 @@ +name: pathml + +channels: + - conda-forge + - pytorch + +dependencies: + - python<=3.10 + - pip==23.3.2 + - numpy==1.23.5 + - scipy<=1.11.4 + - scikit-image<=0.22.0 + - matplotlib<=3.8.2 + - openjdk<=18.0.0 + - h5py==3.10.0 + - dask<=2023.12.1 + - pydicom==2.4.4 + - pytest==7.4.3 + - pre-commit<=3.6.0 + - coverage==7.3.4 + - networkx<=3.2.1 + - pip: + - torch==1.13.1 + - python-bioformats==4.0.7 + - python-javabridge==4.0.3 + - protobuf==3.20.3 + - onnx==1.15.0 + - onnxruntime==1.16.3 + - opencv-contrib-python==4.8.1.78 + - openslide-python==1.3.1 + - scanpy==1.9.6 + - anndata<=0.10.3 + - tqdm==4.66.1 + - loguru==0.7.2 + - pandas<=2.1.4 + - torch-geometric==2.3.1 + - jpype1==1.4.1 From eb80e051ab0acb85309c40067964d60d468cc8d3 Mon Sep 17 00:00:00 2001 From: Varun Ullanat Date: Tue, 5 Mar 2024 18:20:06 +0000 Subject: [PATCH 2/8] Minor edits in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ca419bb7..396c83ee 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ We recommend using [Conda](https://conda.io/projects/conda/en/latest/user-guide/ #### Installing Conda -If you don't have Conda installed, you can download Miniconda [here]. (https://docs.conda.io/en/latest/miniconda.html) +If you don't have Conda installed, you can download Miniconda [here](https://docs.conda.io/en/latest/miniconda.html). #### Updating Conda and Using libmamba (Optional) From b9cd2677b35bba13d84353f5f42cce0e8f5a2928 Mon Sep 17 00:00:00 2001 From: Varun Ullanat Date: Tue, 5 Mar 2024 18:21:31 +0000 Subject: [PATCH 3/8] Removed deepcell in test environment --- requirements/environment_test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements/environment_test.yml b/requirements/environment_test.yml index 27c30a8e..709f30d3 100644 --- a/requirements/environment_test.yml +++ b/requirements/environment_test.yml @@ -24,7 +24,6 @@ dependencies: - python-bioformats==4.0.7 - python-javabridge==4.0.3 - protobuf==3.20.3 - - deepcell<=0.12.7 - onnx==1.15.0 - onnxruntime==1.16.3 - opencv-contrib-python==4.8.1.78 From 0cb2621a1da676dbf10bc8ec08a9d2f9e601e4d6 Mon Sep 17 00:00:00 2001 From: jamesgwen Date: Wed, 6 Mar 2024 09:43:53 -0800 Subject: [PATCH 4/8] adding mesmer to the rst documentation --- docs/source/api_inference_reference.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/source/api_inference_reference.rst b/docs/source/api_inference_reference.rst index 5b8c2eeb..4c6a7ca4 100644 --- a/docs/source/api_inference_reference.rst +++ b/docs/source/api_inference_reference.rst @@ -21,6 +21,11 @@ RemoteTestHoverNet Class .. autoapiclass:: pathml.inference.RemoteTestHoverNet +RemoteMesmer Class +------------------------ + +.. autoapiclass:: pathml.inference.RemoteMesmer + Helper functions ^^^^^^^^^^^^^^^^ From 79e543dd3a3c9e393f77433dec4da25d315df886 Mon Sep 17 00:00:00 2001 From: jamesgwen Date: Wed, 6 Mar 2024 09:44:37 -0800 Subject: [PATCH 5/8] adjusting doc strings for the memser class --- pathml/inference/inference.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/pathml/inference/inference.py b/pathml/inference/inference.py index 5ddc474f..89a51a12 100644 --- a/pathml/inference/inference.py +++ b/pathml/inference/inference.py @@ -442,17 +442,17 @@ class RemoteMesmer(Inference): Nature biotechnology. 2022 Apr;40(4):555-65. Args: - model_path (str): temp file name to download onnx from huggingface, do not change - input_name (str): name of the input the ONNX model accepts, default = "data", do not change - num_classes (int): number of classes you are predicting, do not change - model_type (str): type of model, e.g. "segmentation", do not change - local (bool): True if the model is stored locally, default = "True", do not change - nuclear_channel(int): channel that defines cell nucleus - cytoplasm_channel(int): channel that defines cell membrane or cytoplasm - image_resolution(float): pixel resolution of image in microns. Currently only supports 0.5 - preprocess_kwargs(dict): keyword arguemnts to pass to pre-processing function - postprocess_kwargs_nuclear(dict): keyword arguments to pass to post-processing function - postprocess_kwargs_whole_cell(dict): keyword arguments to pass to post-processing function + model_path (str): temp file name to download onnx from huggingface, do not change + input_name (str): name of the input the ONNX model accepts, default = "data", do not change + num_classes (int): number of classes you are predicting, do not change + model_type (str): type of model, e.g. "segmentation", do not change + local (bool): True if the model is stored locally, default = "True", do not change + nuclear_channel(int): channel that defines cell nucleus + cytoplasm_channel(int): channel that defines cell membrane or cytoplasm + image_resolution(float): pixel resolution of image in microns. Currently only supports 0.5 + preprocess_kwargs(dict): keyword arguemnts to pass to pre-processing function + postprocess_kwargs_nuclear(dict): keyword arguments to pass to post-processing function + postprocess_kwargs_whole_cell(dict): keyword arguments to pass to post-processing function """ def __init__( From 39828b5ce5cf4dab3ab9a98273ab1443d654519f Mon Sep 17 00:00:00 2001 From: jamesgwen Date: Wed, 6 Mar 2024 09:57:53 -0800 Subject: [PATCH 6/8] adding new inference notebook --- examples/InferenceOnnx_tutorial.ipynb | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/examples/InferenceOnnx_tutorial.ipynb b/examples/InferenceOnnx_tutorial.ipynb index 6c3fb201..aa7750d4 100644 --- a/examples/InferenceOnnx_tutorial.ipynb +++ b/examples/InferenceOnnx_tutorial.ipynb @@ -240,7 +240,24 @@ " 'model_input_notes': 'Accepts tiles of 256 x 256',\n", " 'model_output_notes': None,\n", " 'citation': 'Pocock J, Graham S, Vu QD, Jahanifar M, Deshpande S, Hadjigeorghiou G, Shephard A, Bashir RM, Bilal M, Lu W, Epstein D. TIAToolbox as an end-to-end library for advanced tissue image analytics. Communications medicine. 2022 Sep 24;2(1):120.'}\n", - " ```" + " ```\n", + "\n", + "
\n", + "\n", + "- `RemoteMesmer` \n", + " - This class inherits from `Inference` and is hosted on `HuggingFace`. \n", + " - `local` is automatically set to `False` \n", + " - This model is from [Deepcell](https://github.com/vanvalenlab/deepcell-tf/blob/master/deepcell/applications/mesmer.py)\n", + " - Greenwald NF, Miller G, Moen E, Kong A, Kagel A, Dougherty T, Fullaway CC, McIntosh BJ, Leow KX, Schwartz MS, Pavelchek C. Whole-cell segmentation of tissue images with human-level performance using large-scale data annotation and deep learning. Nature biotechnology. 2022 Apr;40(4):555-65.\n", + " - Its `model_card` is:\n", + " - ```python \n", + " {'name': \"Deepcell's Mesmer\",\n", + " 'num_classes': 3,\n", + " 'model_type': 'Segmentation',\n", + " 'notes': None,\n", + " 'model_input_notes': 'Accepts tiles of 256 x 256',\n", + " 'model_output_notes': None,\n", + " 'citation': 'Greenwald NF, Miller G, Moen E, Kong A, Kagel A, Dougherty T, Fullaway CC, McIntosh BJ, Leow KX, Schwartz MS, Pavelchek C. Whole-cell segmentation of tissue images with human-level performance using large-scale data annotation and deep learning. Nature biotechnology. 2022 Apr;40(4):555-65.'}" ] }, { From 747b79614f1859a46e1b71da9a8d1bcb47aafc8f Mon Sep 17 00:00:00 2001 From: Sreekar Reddy Puchala Date: Wed, 6 Mar 2024 20:50:02 +0000 Subject: [PATCH 7/8] Merging changes from dev_graph and debugged a testcase --- pathml/preprocessing/tilestitcher.py | 8 +++- .../preprocessing_tests/test_tilestitcher.py | 44 +++++++++++-------- 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/pathml/preprocessing/tilestitcher.py b/pathml/preprocessing/tilestitcher.py index 9c00e30b..74917567 100644 --- a/pathml/preprocessing/tilestitcher.py +++ b/pathml/preprocessing/tilestitcher.py @@ -123,7 +123,11 @@ def _import_qupath_classes(self): def format_jvm_options(qupath_jars, memory): memory_option = f"-Xmx{memory}" formatted_classpath = [ - path.replace("/", "\\") if platform.system() == "Windows" else path + ( + str(Path(path).as_posix()) + if platform.system() != "Windows" + else str(Path(path)) + ) for path in qupath_jars ] class_path_option = "-Djava.class.path=" + os.pathsep.join(formatted_classpath) @@ -171,6 +175,7 @@ def setup_bfconvert(self, bfconvert_dir): try: if not tools_dir.exists(): + tools_dir.mkdir(parents=True, exist_ok=True) if not bftools_dir.exists(): @@ -204,6 +209,7 @@ def setup_bfconvert(self, bfconvert_dir): zipfile.BadZipFile, PermissionError, subprocess.CalledProcessError, + OSError, ) as e: raise BFConvertSetupError(f"Error setting up bfconvert: {e}") diff --git a/tests/preprocessing_tests/test_tilestitcher.py b/tests/preprocessing_tests/test_tilestitcher.py index 30067cc3..319a3236 100644 --- a/tests/preprocessing_tests/test_tilestitcher.py +++ b/tests/preprocessing_tests/test_tilestitcher.py @@ -1,5 +1,11 @@ +""" +Copyright 2021, Dana-Farber Cancer Institute and Weill Cornell Medicine +License: GNU GPL 2.0 +""" + import glob import os +import platform import subprocess import tempfile import urllib @@ -122,7 +128,7 @@ def test_format_jvm_options_memory(memory, expected_memory_option, tile_stitcher @pytest.mark.exclude @pytest.mark.parametrize( - "qupath_jars, expected_classpath", + "qupath_jars, expected_classpath_suffix", [ ([], ""), ( @@ -131,16 +137,22 @@ def test_format_jvm_options_memory(memory, expected_memory_option, tile_stitcher ), ( ["C:\\path\\to\\jar1.jar", "C:\\path\\to\\jar2.jar"], - "C:\\path\\to\\jar1.jar;C:\\path\\to\\jar2.jar", + "C:\\path\\to\\jar1.jar;C:\\path\\to\\jar2.jar", # Adjusted to use backslashes and semicolon for Windows ), ], ) def test_format_jvm_options_classpath( - qupath_jars, expected_classpath, tile_stitcher, monkeypatch + qupath_jars, expected_classpath_suffix, tile_stitcher, monkeypatch ): - monkeypatch.setattr(os, "pathsep", ";" if os.name == "nt" else ":") + os_name = "nt" if any("C:\\" in jar for jar in qupath_jars) else "posix" + monkeypatch.setattr( + platform, "system", lambda: "Windows" if os_name == "nt" else "Linux" + ) + monkeypatch.setattr(os, "pathsep", ";" if os_name == "nt" else ":") + _, class_path_option = tile_stitcher.format_jvm_options(qupath_jars, "512m") - expected_classpath = "-Djava.class.path=" + os.pathsep.join(qupath_jars) + expected_classpath = "-Djava.class.path=" + expected_classpath_suffix + assert class_path_option == expected_classpath @@ -298,10 +310,12 @@ def test_bfconvert_version_output(tile_stitcher, bfconvert_setup, capsys): ), "bfconvert version not printed correctly" -@pytest.mark.exclude -def test_permission_error_on_directory_creation(tile_stitcher): - with pytest.raises(BFConvertSetupError): - tile_stitcher.setup_bfconvert("/fake/path") +# @pytest.mark.exclude +# def test_permission_error_on_directory_creation(tile_stitcher): +# with patch("pathlib.Path.mkdir", side_effect=OSError("Simulated directory creation failure")): +# with pytest.raises(BFConvertSetupError) as exc_info: +# tile_stitcher.setup_bfconvert("/fake/path") +# assert "directory creation failure" in str(exc_info.value) @pytest.mark.exclude @@ -455,6 +469,8 @@ def test_run_bfconvert_no_delete_original(tile_stitcher, capsys): # Check if the original file still exists assert os.path.exists(stitched_image_path) + if os.path.exists(stitched_image_path): + os.unlink(stitched_image_path) @pytest.mark.exclude @@ -613,16 +629,6 @@ def test_collect_tif_files_invalid_input(tile_stitcher): assert "Invalid input for collecting .tif files:" in str(exc_info.value) -@pytest.mark.exclude -@patch("os.chmod", side_effect=PermissionError("Permission denied")) -def test_setup_bfconvert_permission_error(mock_chmod, tile_stitcher, bfconvert_dir): - with pytest.raises(BFConvertSetupError) as exc_info: - tile_stitcher.setup_bfconvert(bfconvert_dir) - assert "Permission error on setting executable flag: Permission denied" in str( - exc_info.value - ) - - @pytest.mark.exclude def test_parse_region_missing_tags(tile_stitcher): From c7ec88388dc96c757adc33d6b6626f4ea6f1d75f Mon Sep 17 00:00:00 2001 From: Sreekar Reddy Puchala Date: Wed, 6 Mar 2024 20:55:22 +0000 Subject: [PATCH 8/8] Cleaned up code --- tests/preprocessing_tests/test_tilestitcher.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tests/preprocessing_tests/test_tilestitcher.py b/tests/preprocessing_tests/test_tilestitcher.py index 319a3236..ab20eeb8 100644 --- a/tests/preprocessing_tests/test_tilestitcher.py +++ b/tests/preprocessing_tests/test_tilestitcher.py @@ -310,14 +310,6 @@ def test_bfconvert_version_output(tile_stitcher, bfconvert_setup, capsys): ), "bfconvert version not printed correctly" -# @pytest.mark.exclude -# def test_permission_error_on_directory_creation(tile_stitcher): -# with patch("pathlib.Path.mkdir", side_effect=OSError("Simulated directory creation failure")): -# with pytest.raises(BFConvertSetupError) as exc_info: -# tile_stitcher.setup_bfconvert("/fake/path") -# assert "directory creation failure" in str(exc_info.value) - - @pytest.mark.exclude @pytest.fixture def mock_subprocess(monkeypatch):