From 45a6aba2bc3541b757c01a61ea49cc5f62cdacd1 Mon Sep 17 00:00:00 2001 From: Mittmich Date: Mon, 11 Jan 2021 15:53:38 +0100 Subject: [PATCH 1/7] removed mirny tools --- Dockerfile | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/Dockerfile b/Dockerfile index cf6706a..a069474 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,32 +44,10 @@ WORKDIR /home RUN source activate ngs_base &&\ # Creates a file into the container that logs which version conda installs and all pip installs conda list > software_versions_conda.txt &&\ - # Install mirnylabtools - # bioframe - githash=`git ls-remote https://github.com/mirnylab/bioframe.git | grep HEAD | cut -f 1` &&\ - pip install git+git://github.com/mirnylab/bioframe@$githash &&\ - echo "# pip install git+git://github.com/mirnylab/bioframe@$githash" >> software_versions_git.txt &&\ - # cooltools - githash=`git ls-remote https://github.com/mirnylab/cooltools.git | grep HEAD | cut -f 1` &&\ - pip install git+git://github.com/mirnylab/cooltools@$githash &&\ - echo "# pip install git+git://github.com/mirnylab/cooltools@$githash" >> software_versions_git.txt &&\ - # pairlib - githash=`git ls-remote https://github.com/mirnylab/pairlib.git | grep HEAD | cut -f 1` &&\ - pip install git+git://github.com/mirnylab/pairlib@$githash &&\ - echo "# pip install git+git://github.com/mirnylab/pairlib@$githash" >> software_versions_git.txt &&\ - # Install gerlich repos and safe the latest git hash # ngs githash=`git ls-remote https://github.com/gerlichlab/ngs.git | grep HEAD | cut -f 1` &&\ pip install git+git://github.com/gerlichlab/ngs@$githash &&\ echo "# pip install git+git://github.com/gerlichlab/ngs@$githash" >> software_versions_git.txt &&\ - # cooler_ontad - githash=`git ls-remote https://github.com/cchlanger/cooler_ontad.git | grep HEAD | cut -f 1` &&\ - pip install git+git://github.com/cchlanger/cooler_ontad@$githash &&\ - echo "# pip install git+git://github.com/cchlanger/cooler_ontad@$githash" >> software_versions_git.txt &&\ - # higlassup - githash=`git ls-remote https://github.com/Mittmich/higlassupload.git | grep HEAD | cut -f 1` &&\ - pip install git+git://github.com/Mittmich/higlassupload.git@$githash &&\ - echo "# pip install git+git://github.com/Mittmich/higlassupload.git@$githash" >> software_versions_git.txt ENV PATH="/home/anaconda3/envs/ngs_base/bin/:${PATH}" From cd22763a5b911890f78c8269cbd42cdac34f8b41 Mon Sep 17 00:00:00 2001 From: Mittmich Date: Mon, 11 Jan 2021 16:04:08 +0100 Subject: [PATCH 2/7] fixed typo --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a069474..c7c0f4d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -47,7 +47,7 @@ RUN source activate ngs_base &&\ # ngs githash=`git ls-remote https://github.com/gerlichlab/ngs.git | grep HEAD | cut -f 1` &&\ pip install git+git://github.com/gerlichlab/ngs@$githash &&\ - echo "# pip install git+git://github.com/gerlichlab/ngs@$githash" >> software_versions_git.txt &&\ + echo "# pip install git+git://github.com/gerlichlab/ngs@$githash" >> software_versions_git.txt ENV PATH="/home/anaconda3/envs/ngs_base/bin/:${PATH}" From 3e0d250093fdadb7127f6ceb690c65fe34452f1f Mon Sep 17 00:00:00 2001 From: Mittmich Date: Tue, 12 Jan 2021 08:54:58 +0100 Subject: [PATCH 3/7] took out ngs to be able to install newer versions faster --- Dockerfile | 9 --------- 1 file changed, 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index c7c0f4d..7863bfe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,15 +40,6 @@ RUN conda env create -f /temp/install/flask.yml WORKDIR /home -# Install bioframe, cooltools and pairlib as well as our own tools -RUN source activate ngs_base &&\ - # Creates a file into the container that logs which version conda installs and all pip installs - conda list > software_versions_conda.txt &&\ - # ngs - githash=`git ls-remote https://github.com/gerlichlab/ngs.git | grep HEAD | cut -f 1` &&\ - pip install git+git://github.com/gerlichlab/ngs@$githash &&\ - echo "# pip install git+git://github.com/gerlichlab/ngs@$githash" >> software_versions_git.txt - ENV PATH="/home/anaconda3/envs/ngs_base/bin/:${PATH}" CMD /bin/bash \ No newline at end of file From 260063b3e8264607a686404a4419454877022394 Mon Sep 17 00:00:00 2001 From: Michael Mitter Date: Wed, 19 May 2021 15:50:13 +0200 Subject: [PATCH 4/7] bumped python version since pickle issue (too large struct) might be fixed there --- flask.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flask.yml b/flask.yml index b681975..905b10d 100644 --- a/flask.yml +++ b/flask.yml @@ -5,7 +5,7 @@ channels: - defaults dependencies: # python - - python=3.7.6 + - python=3.8.0 # user required tools: - cooler >= 0.8.8 - numpy >= 1.18.5 From 75f65b63562c64b73014b9e4d01ef6518e2070e6 Mon Sep 17 00:00:00 2001 From: Michael Mitter Date: Fri, 21 May 2021 16:55:43 +0200 Subject: [PATCH 5/7] prototyping smaller image for pipeline --- Dockerfile | 53 ++++++++++++++++++++++------------------------------ flask.yml | 39 -------------------------------------- ngs_base.yml | 23 ++--------------------- 3 files changed, 24 insertions(+), 91 deletions(-) delete mode 100644 flask.yml diff --git a/Dockerfile b/Dockerfile index 7863bfe..9bd2411 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,45 +1,36 @@ -FROM docker.io/ubuntu:bionic -ARG DEBIAN_FRONTEND=noninteractive -# Can not use /temp on our cluster -WORKDIR /temp -COPY . /temp/install/ +FROM continuumio/miniconda3:4.9.2 +ENV DEBIAN_FRONTEND=noninteractive ENV LANG C.UTF-8 ENV LC_ALL C.UTF-8 # Update and create base image RUN apt-get update -y &&\ - apt-get install apt-utils -y &&\ - apt-get install -y file bzip2 default-jre gcc g++ git make ssh unzip wget libz-dev iproute2 curl iputils-ping &&\ + apt-get install -y gcc g++ make libz-dev &&\ apt-get clean -# Install Anaconda -SHELL ["/bin/bash", "-c"] -RUN wget -q https://repo.anaconda.com/archive/Anaconda3-2019.10-Linux-x86_64.sh &&\ - bash Anaconda3-2019.10-Linux-x86_64.sh -b -p /home/anaconda3 -ENV PATH="/home/anaconda3/bin:${PATH}" -RUN conda update -y conda &&\ - conda update -y conda-build - -# Install OnTAD -RUN cd /opt && git clone https://github.com/anlin00007/OnTAD.git &&\ - cd OnTAD &&\ - git checkout 3da5d9a4569b1f316d4508e60781f22f338f68b1 -RUN cd /opt/OnTAD/src && make clean && make ENV PATH="/opt/OnTAD/src:${PATH}" -# Install fastqc -RUN cd /opt && wget http://www.bioinformatics.babraham.ac.uk/projects/fastqc/fastqc_v0.11.9.zip &&\ - unzip /opt/fastqc_v0.11.9.zip &&\ - chmod +x /opt/FastQC/fastqc -ENV PATH="/opt/FastQC:${PATH}" - # Install ngs_base environment -RUN conda env create -f /temp/install/ngs_base.yml -# Install flask environment -RUN conda env create -f /temp/install/flask.yml +ADD ngs_base.yml /temp/install/ + +# install mamba +RUN conda install mamba -n base -c conda-forge &&\ + mamba env update -n base --f /temp/install/ngs_base.yml &&\ + mamba list > software_versions_conda.txt &&\ + # Install mirnylabtools + # bioframe + githash=`git ls-remote https://github.com/mirnylab/bioframe.git | grep HEAD | cut -f 1` &&\ + pip install git+git://github.com/mirnylab/bioframe@$githash &&\ + echo "# pip install git+git://github.com/mirnylab/bioframe@$githash" >> software_versions_git.txt &&\ + # cooltools + githash=`git ls-remote https://github.com/mirnylab/cooltools.git | grep HEAD | cut -f 1` &&\ + pip install git+git://github.com/mirnylab/cooltools@$githash &&\ + echo "# pip install git+git://github.com/mirnylab/cooltools@$githash" >> software_versions_git.txt &&\ + # pairlib + githash=`git ls-remote https://github.com/mirnylab/pairlib.git | grep HEAD | cut -f 1` &&\ + pip install git+git://github.com/mirnylab/pairlib@$githash &&\ + echo "# pip install git+git://github.com/mirnylab/pairlib@$githash" >> software_versions_git.txt WORKDIR /home -ENV PATH="/home/anaconda3/envs/ngs_base/bin/:${PATH}" - CMD /bin/bash \ No newline at end of file diff --git a/flask.yml b/flask.yml deleted file mode 100644 index 905b10d..0000000 --- a/flask.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: flask -channels: - - bioconda - - conda-forge - - defaults -dependencies: - # python - - python=3.8.0 - # user required tools: - - cooler >= 0.8.8 - - numpy >= 1.18.5 - - sortedcontainers >= 2.2.2 - - matplotlib >= 3.2.2 - - multiprocess >= 0.70.10 - - pairtools >= 0.3.0 - - pandas >= 1.0.5 - - pybedtools >= 0.8.1 - - pybigwig >=0.3.17 - - pysam >= 0.16.0.1 - - scikit-learn >= 0.23.1 - - scipy >= 1.5.0 - - seaborn >= 0.10.1 - - statsmodels >= 0.11.1 - # requirements for clodius - - cython >= 0.29.20 - # requirements for bioframe: - - six >=1.15.0 - - requests >= 2.24.0 - - pairix >= 0.3.7 - # requirements for cooltools: - - h5py >= 2.10.0 - - numba >= 0.48.0 - - dask >= 2.19.0 - - click >= 7.1.2 - - pytest >= 5.4.3 - # packages to facility development with vscode inside the container - - pylama >= 7.7.1 - - pylint >= 2.5.3 - - black >= 19.10b0 \ No newline at end of file diff --git a/ngs_base.yml b/ngs_base.yml index 1fdce76..d3b02ab 100644 --- a/ngs_base.yml +++ b/ngs_base.yml @@ -5,8 +5,9 @@ channels: - defaults dependencies: # newest python - - python=3.7.6 + - python=3.8.0 # user required tools: + - fastqc >= 0.11.9 - cooler >= 0.8.8 - numpy >= 1.18.5 - sortedcontainers >= 2.2.2 @@ -14,10 +15,7 @@ dependencies: - multiprocess >= 0.70.10 - pairtools >= 0.3.0 - pandas >= 1.0.5 - - pybedtools >= 0.8.1 - - pybigwig >=0.3.17 - pysam >= 0.16.0.1 - - scikit-learn >= 0.23.1 - scipy >= 1.5.0 - seaborn >= 0.10.1 - statsmodels >= 0.11.1 @@ -30,32 +28,15 @@ dependencies: - requests >= 2.24.0 - pairix >= 0.3.7 # requirements for cooltools: - # - numpy - # - cython - # - matplotlib - # - pandas - # - scipy - # - cooler - # - scikit-learn - h5py >= 2.10.0 - numba >= 0.48.0 - dask >= 2.19.0 - click >= 7.1.2 - pytest >= 5.4.3 - # packages to facility development with vscode inside the container - - pylama >= 7.7.1 - - pylint >= 2.5.3 - - black >= 19.10b0 # parquet support - pyensembl >= 1.8.7 - pyarrow >= 0.17.1 - fastparquet >= 0.4.0 - # jupyter support - - ipywidgets >= 7.5.1 - - ipykernel >= 5.3.0 - - jupyterlab >= 2.1.5 - # loop caller - - chromosight >= 1.1.4 # for nextflowpipeline - nextflow >= 20.04.1 - bwa >= 0.7.17 From 384d39c1e8eb1ad6516a6fbfe4e67e8e4722bc9e Mon Sep 17 00:00:00 2001 From: Mittmich Date: Tue, 25 May 2021 11:07:22 +0200 Subject: [PATCH 6/7] removed unnecessary env variable --- Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9bd2411..6bbf335 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,8 +8,6 @@ RUN apt-get update -y &&\ apt-get install -y gcc g++ make libz-dev &&\ apt-get clean -ENV PATH="/opt/OnTAD/src:${PATH}" - # Install ngs_base environment ADD ngs_base.yml /temp/install/ From 2352fcacf9dbb47f8f5ae309cf7f2846bf1f622d Mon Sep 17 00:00:00 2001 From: Mittmich Date: Tue, 25 May 2021 11:19:37 +0200 Subject: [PATCH 7/7] edited readme --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 070f223..923712f 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,11 @@ This container will be kept updated and future versions will include additional # Dockerhub and Version This container can be found on [dockerhub](https://hub.docker.com/r/gerlichlab/scshic_docker). + +## Version 1.5 + +This version removes things not needed by our [scsHi-C pipeline](https://github.com/gerlichlab/scshic_pipeline) and tries to minimize container size. + ## Version 1.3 This version is for our next project. Adds all dependencies for a flask server, but not flask itself.