Add github action to build a Docker image and update for Nextflow 21.04.1

.github/scripts/install_singularity.sh

@@ -22,9 +22,9 @@ go version
 # install Singularity
 export VERSION=3.5.3
 echo Install Singularity version $VERSION .. >> artifacts/test_artifact.log
-wget https://github.com/sylabs/singularity/releases/download/v${VERSION}/singularity-${VERSION}.tar.gz
+wget https://codeload.github.com/sylabs/singularity/tar.gz/refs/tags/v${VERSION} -O singularity-${VERSION}.tar.gz
 tar -xzf singularity-${VERSION}.tar.gz
-cd singularity
+cd singularity-${VERSION}
 ./mconfig
 make -C builddir
 sudo make -C builddir install

.github/scripts/test_PR_against_release.sh

@@ -6,7 +6,7 @@ export PATH=/opt/conda/bin:$PATH
 singularity --version
 # write test log as github Action artifact
 echo Nextflow run current PR in --illumina mode.. >> artifacts/test_artifact.log
-NXF_VER=20.03.0-edge nextflow run ./main.nf \
+NXF_VER=21.04.1 nextflow run ./main.nf \
        -profile singularity \
        --directory $PWD/.github/data/fastqs/ \
        --illumina \
@@ -25,7 +25,7 @@ git checkout tags/v1.1.1
 sed -i s'/cpus = 4/cpus = 2/'g conf/resources.config
 ln -s ../*.sif ./
 echo Nextflow run previous release in --illumina mode.. >> ../artifacts/test_artifact.log
-NXF_VER=20.03.0-edge nextflow run ./main.nf \
+NXF_VER=21.04.1 nextflow run ./main.nf \
        -profile singularity \
        --directory $PWD/../.github/data/fastqs/ \
        --illumina \

.github/scripts/test_bed_ref_input.sh

@@ -14,7 +14,7 @@ echo bed file: $BED_FILE >> artifacts/test_artifact.log
 # run current pull request code
 singularity --version
 echo Nextflow run --illumina mode with --ref, --bed .. >> artifacts/test_artifact.log
-NXF_VER=20.03.0-edge nextflow run ./main.nf \
+NXF_VER=21.04.1 nextflow run ./main.nf \
        -profile singularity \
        --ref $REF_FILE \
        --bed $BED_FILE \
@@ -58,7 +58,7 @@ rm $REF_FILE
 ln -s $REAL_REF $REF_FILE
 
 echo Nextflow run --illumina mode with symlinked --ref, --bed .. >> artifacts/test_artifact.log
-NXF_VER=20.03.0-edge nextflow run ./main.nf \
+NXF_VER=21.04.1 nextflow run ./main.nf \
        -profile singularity \
        --ref $REF_FILE \
        --bed $BED_FILE \

.github/scripts/test_conda_cache.sh

@@ -10,7 +10,7 @@ export REPO=$PWD
 echo REPO=$REPO >> artifacts/test_artifact.log
 cd ..
 echo PWD=$PWD >> $REPO/artifacts/test_artifact.log
-NXF_VER=20.03.0-edge nextflow run $REPO \
+NXF_VER=21.04.1 nextflow run $REPO \
        -profile conda \
        --cache $REPO/conda_cache_dir \
        --directory $REPO/.github/data/fastqs/ \
@@ -25,7 +25,7 @@ cat .nextflow.log | grep 'Conda create complete env=/home/runner/work/ncov2019-a
 rm -rf results && rm -rf work && rm -rf .nextflow*
 # second NF run will use the conda env created in the previous run
 echo re-run pipeline with conda --cache.. >> $REPO/artifacts/test_artifact.log
-NXF_VER=20.03.0-edge nextflow run $REPO \
+NXF_VER=21.04.1 nextflow run $REPO \
        -profile conda \
        --cache $REPO/conda_cache_dir \
        --directory $REPO/.github/data/fastqs/ \

.github/scripts/test_cram_input.sh

@@ -14,7 +14,7 @@ echo bed file: $BED_FILE
 sed -i s'/cpus = 4/cpus = 2/'g conf/coguk/sanger.config
 singularity --version
 echo Nextflow run --illumina mode with --ref, --bed and --cram.. >> artifacts/test_artifact.log
-NXF_VER=20.03.0-edge nextflow run ./main.nf \
+NXF_VER=21.04.1 nextflow run ./main.nf \
        -profile sanger,singularity \
        --ref $REF_FILE \
        --bed $BED_FILE \

.github/scripts/test_cram_output.sh

@@ -14,7 +14,7 @@ echo bed file: $BED_FILE
 sed -i s'/cpus = 4/cpus = 2/'g conf/coguk/sanger.config
 singularity --version
 echo Nextflow run --illumina mode with --ref, --bed --cram and outCram... >> artifacts/test_artifact.log
-NXF_VER=20.03.0-edge nextflow run ./main.nf \
+NXF_VER=21.04.1 nextflow run ./main.nf \
        -profile sanger,singularity \
        --ref $REF_FILE \
        --bed $BED_FILE \

.github/scripts/test_nanopore_pipelines.sh

@@ -6,7 +6,7 @@ export PATH=/opt/conda/bin:$PATH
 singularity --version
 # write test log as github Action artifact
 echo "Nextflow run current PR in --nanopolish mode (no barcodes).." >> artifacts/test_artifact.log
-NXF_VER=20.03.0-edge nextflow run ./main.nf \
+NXF_VER=21.04.1 nextflow run ./main.nf \
        -profile singularity \
        --nanopolish \
        --sequencing_summary $PWD/.github/data/nanopore/20200311_1427_X4_FAK72834_a3787181/sequencing_summary_FAK72834_298b7829.txt \
@@ -17,7 +17,7 @@ cp .nextflow.log artifacts/nanopolish.nextflow.log
 rm -rf results && rm -rf work && rm -rf .nextflow*
 
 echo "Nextflow run current PR in --nanopolish mode (barcodes).." >> artifacts/test_artifact.log
-NXF_VER=20.03.0-edge nextflow run ./main.nf \
+NXF_VER=21.04.1 nextflow run ./main.nf \
        -profile singularity \
        --nanopolish \
        --sequencing_summary $PWD/.github/data/nanopore/20200311_1427_X1_FAK72834_a3787181/sequencing_summary_FAK72834_298b7829.txt \
@@ -28,7 +28,7 @@ cp .nextflow.log artifacts/nanopolish_barcodes.nextflow.log
 rm -rf results && rm -rf work && rm -rf .nextflow*
 
 echo "Nextflow run current PR in --medaka mode (no barcodes).." >> artifacts/test_artifact.log
-NXF_VER=20.03.0-edge nextflow run ./main.nf \
+NXF_VER=21.04.1 nextflow run ./main.nf \
        -profile singularity \
        --medaka \
        --basecalled_fastq $PWD/.github/data/nanopore/20200311_1427_X4_FAK72834_a3787181/fastq_pass/ \
@@ -37,7 +37,7 @@ cp .nextflow.log artifacts/medaka.nextflow.log
 rm -rf results && rm -rf work && rm -rf .nextflow*
 
 echo "Nextflow run current PR in --medaka mode (barcodes).." >> artifacts/test_artifact.log
-NXF_VER=20.03.0-edge nextflow run ./main.nf \
+NXF_VER=21.04.1 nextflow run ./main.nf \
        -profile singularity \
        --medaka \
        --basecalled_fastq $PWD/.github/data/nanopore/20200311_1427_X1_FAK72834_a3787181/fastq_pass/ \

.github/scripts/test_sanger_profile.sh

@@ -6,7 +6,7 @@ export PATH=/opt/conda/bin:$PATH
 # there are only 2 available cpus in the github runner execution
 sed -i s'/cpus = 4/cpus = 2/'g conf/coguk/sanger.config
 echo run pipeline in --illumina mode with --sanger profile.. >> artifacts/test_artifact.log
-NXF_VER=20.03.0-edge nextflow run ./main.nf \
+NXF_VER=21.04.1 nextflow run ./main.nf \
        -profile sanger,singularity \
        --directory $PWD/.github/data/fastqs/ \
        --illumina \

.github/scripts/test_typing.sh

@@ -6,7 +6,7 @@ export PATH=/opt/conda/bin:$PATH
 singularity --version
 # write test log as github Action artifact
 echo "Nextflow run current PR in --nanopolish mode with typing.." >> artifacts/test_artifact.log
-NXF_VER=20.03.0-edge nextflow run ./main.nf \
+NXF_VER=21.04.1 nextflow run ./main.nf \
        -profile singularity \
        --gff $PWD/typing/MN908947.3.gff \
        --yaml $PWD/typing/SARS-CoV-2.types.yaml \
@@ -19,7 +19,7 @@ cp .nextflow.log artifacts/nanopolish_typing.nextflow.log
 rm -rf results && rm -rf work && rm -rf .nextflow*
 
 echo "Nextflow run current PR in --medaka mode with typing .." >> artifacts/test_artifact.log
-NXF_VER=20.03.0-edge nextflow run ./main.nf \
+NXF_VER=21.04.1 nextflow run ./main.nf \
        -profile singularity \
        --medaka \
        --gff $PWD/typing/MN908947.3.gff \
@@ -30,7 +30,7 @@ cp .nextflow.log artifacts/medaka_typing.nextflow.log
 rm -rf results && rm -rf work && rm -rf .nextflow*
 
 echo Nextflow run current PR in --illumina mode with typing.. >> artifacts/test_artifact.log
-NXF_VER=20.03.0-edge nextflow run ./main.nf \
+NXF_VER=21.04.1 nextflow run ./main.nf \
        -profile singularity \
        --gff $PWD/typing/MN908947.3.gff \
        --yaml $PWD/typing/SARS-CoV-2.types.yaml \

.github/workflows/docker-build-and-push.yml

@@ -0,0 +1,63 @@
+name: Create and publish a Docker image
+
+on:   
+  push:
+    branches:
+      - master
+    # Publish `v1.2.3` tags as releases.
+    tags:
+      - v*
+      - t*
+  # Run tests for any PRs.
+  pull_request:
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  # Push image to GitHub Packages.
+  # See also https://docs.docker.com/docker-hub/builds/
+  push:
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Build image
+        run: |
+          docker build . --file environments/nanopore/Dockerfile --tag $IMAGE_NAME-nanopore --label "runnumber=${GITHUB_RUN_ID}"
+          docker build . --file environments/illumina/Dockerfile --tag $IMAGE_NAME-illumina --label "runnumber=${GITHUB_RUN_ID}"
+
+      - name: Log in to registry
+        if: ${{ github.event_name != 'pull_request' }}
+        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+
+      - name: Push image
+        if: ${{ github.event_name != 'pull_request' }}
+        run: |
+          IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME
+
+          # Change all uppercase to lowercase
+          IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
+          # Strip git ref prefix from version
+          VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
+          # Strip "v" prefix from tag name
+          [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
+          # Use Docker `latest` tag convention
+          [ "$VERSION" == "master" ] && VERSION=latest
+          docker tag $IMAGE_NAME-nanopore $IMAGE_ID:$VERSION-nanopore
+          docker push $IMAGE_ID:$VERSION-nanopore
+          docker tag $IMAGE_NAME-illumina $IMAGE_ID:$VERSION-illumina
+          docker push $IMAGE_ID:$VERSION-illumina
+      - name: Push latest
+        if: ${{ github.event_name != 'pull_request' && startsWith(github.ref, 'refs/tags/v')}}
+        run: |
+          IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME
+          docker tag $IMAGE_NAME-nanopore $IMAGE_ID:latest-nanopore
+          docker push $IMAGE_ID:latest-nanopore
+          docker tag $IMAGE_NAME-illumina $IMAGE_ID:latest-illumina
+          docker push $IMAGE_ID:latest-illumina

.github/workflows/pull_request.yml

@@ -15,18 +15,18 @@ jobs:
       run: mkdir artifacts
     - name: install Singularity
       run: |
-       bash .github/scripts/install_singularity.sh
+        bash .github/scripts/install_singularity.sh
     - name: Build pipeline Singularity containers
       run: |
-       echo $(which singularity)
-       bash scripts/build_singularity_containers.sh
+        echo $(which singularity)
+        bash scripts/build_singularity_containers.sh
     - name: install Conda
       run: bash .github/scripts/install_conda.sh
     - name: install Nextflow via Conda
       run: |
-       export PATH=/opt/conda/bin:$PATH
-       conda install -c bioconda nextflow
-       NXF_VER=20.03.0-edge nextflow -version
+        export PATH=/opt/conda/bin:$PATH
+        conda install -c bioconda nextflow
+        NXF_VER=21.04.1 nextflow -version
     - name: test nanopore pipelines
       run: bash .github/scripts/test_nanopore_pipelines.sh
     - name: test typing functionality
@@ -50,4 +50,4 @@ jobs:
       if: always()
       with:
         name: test_artifacts
-        path: artifacts
+        path: artifacts

conf/base.config

@@ -2,11 +2,11 @@
 
 params{
     // Boilerplate to keep nextflow happy about undefined vars
-    directory = false
-    prefix = false
-    basecalled_fastq = false
-    fast5_pass = false
-    sequencing_summary = false
+    directory = null
+    prefix = null
+    basecalled_fastq = null
+    fast5_pass = null
+    sequencing_summary = null
     ref = false
     bed = false
     gff = false

environments/nanopore/Dockerfile

@@ -13,4 +13,3 @@ apt-get install -y git procps && \
 apt-get clean -y
 COPY --from=condabuild /opt/conda/envs/artic /opt/conda/envs/artic
 ENV PATH=/opt/conda/envs/artic/bin:$PATH
-ENTRYPOINT ["/opt/conda/envs/artic/bin/artic"]

main.nf

@@ -1,7 +1,7 @@
 #!/usr/bin/env nextflow
 
 // enable dsl2
-nextflow.preview.dsl = 2
+nextflow.enable.dsl = 2
 
 // include modules
 include {printHelp} from './modules/help.nf'

modules/artic.nf

@@ -54,10 +54,10 @@ process articMinIONMedaka {
     output:
     file("${sampleName}*")
 
-    tuple sampleName, file("${sampleName}.primertrimmed.rg.sorted.bam"), emit: ptrim
-    tuple sampleName, file("${sampleName}.sorted.bam"), emit: mapped
-    tuple sampleName, file("${sampleName}.consensus.fasta"), emit: consensus_fasta
-    tuple sampleName, file("${sampleName}.pass.vcf.gz"), emit: vcf
+    tuple val(sampleName), file("${sampleName}.primertrimmed.rg.sorted.bam"), emit: ptrim
+    tuple val(sampleName), file("${sampleName}.sorted.bam"), emit: mapped
+    tuple val(sampleName), file("${sampleName}.consensus.fasta"), emit: consensus_fasta
+    tuple val(sampleName), file("${sampleName}.pass.vcf.gz"), emit: vcf
 
     script:
     // Make an identifier from the fastq filename
@@ -102,10 +102,10 @@ process articMinIONNanopolish {
     output:
     file("${sampleName}*")
 
-    tuple sampleName, file("${sampleName}.primertrimmed.rg.sorted.bam"), emit: ptrim
-    tuple sampleName, file("${sampleName}.sorted.bam"), emit: mapped
-    tuple sampleName, file("${sampleName}.consensus.fasta"), emit: consensus_fasta
-    tuple sampleName, file("${sampleName}.pass.vcf.gz"), emit: vcf
+    tuple val(sampleName), file("${sampleName}.primertrimmed.rg.sorted.bam"), emit: ptrim
+    tuple val(sampleName), file("${sampleName}.sorted.bam"), emit: mapped
+    tuple val(sampleName), file("${sampleName}.consensus.fasta"), emit: consensus_fasta
+    tuple val(sampleName), file("${sampleName}.pass.vcf.gz"), emit: vcf
 
     script:
     // Make an identifier from the fastq filename
@@ -144,10 +144,10 @@ process articRemoveUnmappedReads {
     cpus 1
 
     input:
-    tuple(sampleName, path(bamfile))
+    tuple(val(sampleName), path(bamfile))
 
     output:
-    tuple( sampleName, file("${sampleName}.mapped.sorted.bam"))
+    tuple(val(sampleName), file("${sampleName}.mapped.sorted.bam"))
 
     script:
     """