.gitlab-ci.yml

image: quay.io/vgteam/vg_ci_prebake:latest
# Note that we must run in a privileged container for our internal Docker daemon to come up.

variables:
  PYTHONIOENCODING: "utf-8"
  DEBIAN_FRONTEND: "noninteractive"
  TOIL_OWNER_TAG: "shared"

before_script:
  # Configure Docker to use a mirror for Docker Hub and restart the daemon
  # Set the registry as insecure because it is probably cluster-internal over plain HTTP.
  - |
    if [[ ! -z "${DOCKER_HUB_MIRROR}" ]] ; then
        echo "{\"registry-mirrors\": [\"${DOCKER_HUB_MIRROR}\"], \"insecure-registries\": [\"${DOCKER_HUB_MIRROR##*://}\"]}" | sudo tee /etc/docker/daemon.json
        export SINGULARITY_DOCKER_HUB_MIRROR="${DOCKER_HUB_MIRROR}"
    fi
  - startdocker || true
  - docker info
  - cat /etc/hosts
  - mkdir -p ~/.kube && cp "$GITLAB_SECRET_FILE_KUBE_CONFIG" ~/.kube/config
  - mkdir -p ~/.aws && cp "$GITLAB_SECRET_FILE_AWS_CREDENTIALS" ~/.aws/credentials
  - sudo apt-get update
  - sudo apt-get install -y software-properties-common build-essential virtualenv
  - sudo add-apt-repository -y ppa:deadsnakes/ppa
  - sudo apt-get update
  - sudo apt-get install -y tzdata jq python3.6 python3.6-dev python3.7 python3.7-dev python3.8 python3.8-dev

after_script:
  # We need to clean up any files that Toil may have made via Docker that
  # aren't deletable by the Gitlab user. If we don't do this, Gitlab will try
  # and clean them up before running the next job on the runner, fail, and fail
  # that next job.
  - pwd
  - sudo rm -rf tmp
  - stopdocker || true

stages:
  - linting_and_dependencies
  - basic_tests
  - main_tests
  - integration


lint:
  stage: linting_and_dependencies
  script:
    - pwd
    - virtualenv -p python3.6 venv && . venv/bin/activate && make prepare && make develop extras=[all] packages=htcondor
    - make mypy
    - make docs


cwl_dependency_is_stand_alone:
  stage: linting_and_dependencies
  script:
    - pwd
    - virtualenv -p python3.6 venv && . venv/bin/activate && make prepare && make develop extras=[cwl]
    - make test tests=src/toil/test/docs/scriptsTest.py::ToilDocumentationTest::testCwlexample


wdl_dependency_is_stand_alone:
  stage: linting_and_dependencies
  script:
    - pwd
    - virtualenv -p python3.6 venv && . venv/bin/activate && make prepare && make develop extras=[wdl]
    - make test tests=src/toil/test/wdl/toilwdlTest.py::ToilWdlIntegrationTest::testMD5sum

# Python3.6
py36_main:
  stage: basic_tests
  script:
    - pwd
    - virtualenv -p python3.6 venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages=htcondor
    - make test tests=src/toil/test/src
    - make test tests=src/toil/test/utils

py36_appliance_build:
  stage: basic_tests
  script:
    - pwd
    - virtualenv -p python3.6 venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor awscli'
    # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
    - python setup_gitlab_docker.py
    - make push_docker


# Python3.7
py37_batch_systems:
  stage: main_tests
  script:
    - pwd
    - virtualenv -p python3.7 venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor awscli'
    - make test tests=src/toil/test/batchSystems/batchSystemTest.py
    - make test tests=src/toil/test/mesos/MesosDataStructuresTest.py

py37_cwl_v1.0:
  stage: main_tests
  only: []
  script:
    - pwd
    - virtualenv -p python3.7 venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[cwl,aws]
    - mypy --ignore-missing-imports --no-strict-optional $(pwd)/src/toil/cwl/cwltoil.py  # make this a separate linting stage
    - python setup_gitlab_docker.py  # login to increase the docker.io rate limit
    - make test tests=src/toil/test/cwl/cwlTest.py::CWLv10Test

py37_cwl_v1.1:
  stage: main_tests
  only: []
  script:
    - pwd
    - virtualenv -p python3.7 venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[cwl,aws]
    - python setup_gitlab_docker.py  # login to increase the docker.io rate limit
    - make test tests=src/toil/test/cwl/cwlTest.py::CWLv11Test

py37_cwl_v1.2:
  stage: main_tests
  script:
    - pwd
    - virtualenv -p python3.7 venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[cwl,aws]
    - python setup_gitlab_docker.py  # login to increase the docker.io rate limit
    - make test tests=src/toil/test/cwl/cwlTest.py::CWLv12Test

py37_cwl_v1.0_kubernetes:
  stage: main_tests
  only: []
  script:
    - pwd
    - virtualenv -p python3.7 venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[cwl,aws,kubernetes]
    - export TOIL_KUBERNETES_OWNER=toiltest
    - export TOIL_AWS_SECRET_NAME=shared-s3-credentials
    - export TOIL_KUBERNETES_HOST_PATH=/data/scratch
    - export TOIL_WORKDIR=/var/lib/toil
    - export SINGULARITY_CACHEDIR=/var/lib/toil/singularity-cache
    - echo Singularity mirror is ${SINGULARITY_DOCKER_HUB_MIRROR}
    - mkdir -p ${TOIL_WORKDIR}
    - make test tests=src/toil/test/cwl/cwlTest.py::CWLv10Test::test_kubernetes_cwl_conformance
    - make test tests=src/toil/test/cwl/cwlTest.py::CWLv10Test::test_kubernetes_cwl_conformance_with_caching
    
py37_cwl_v1.1_kubernetes:
  stage: main_tests
  only: []
  script:
    - pwd
    - virtualenv -p python3.7 venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[cwl,aws,kubernetes]
    - export TOIL_KUBERNETES_OWNER=toiltest
    - export TOIL_AWS_SECRET_NAME=shared-s3-credentials
    - export TOIL_KUBERNETES_HOST_PATH=/data/scratch
    - export TOIL_WORKDIR=/var/lib/toil
    - export SINGULARITY_CACHEDIR=/var/lib/toil/singularity-cache
    - mkdir -p ${TOIL_WORKDIR}
    - make test tests=src/toil/test/cwl/cwlTest.py::CWLv11Test::test_kubernetes_cwl_conformance
    - make test tests=src/toil/test/cwl/cwlTest.py::CWLv11Test::test_kubernetes_cwl_conformance_with_caching
    
py37_cwl_v1.2_kubernetes:
  stage: main_tests
  script:
    - pwd
    - virtualenv -p python3.7 venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[cwl,aws,kubernetes]
    - export TOIL_KUBERNETES_OWNER=toiltest
    - export TOIL_AWS_SECRET_NAME=shared-s3-credentials
    - export TOIL_KUBERNETES_HOST_PATH=/data/scratch
    - export TOIL_WORKDIR=/var/lib/toil
    - export SINGULARITY_CACHEDIR=/var/lib/toil/singularity-cache
    - mkdir -p ${TOIL_WORKDIR}
    - make test tests=src/toil/test/cwl/cwlTest.py::CWLv12Test

py37_wdl:
  stage: main_tests
  script:
    - pwd
    - apt update && apt install -y default-jre
    - virtualenv -p python3.7 venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all]
    - which java &> /dev/null || { echo >&2 "Java is not installed.  Install java to run these tests."; exit 1; }
    - make test tests=src/toil/test/wdl/toilwdlTest.py  # needs java (default-jre) to run "GATK.jar"
    - make test tests=src/toil/test/wdl/builtinTest.py

py37_jobstore_and_provisioning:
  stage: main_tests
  script:
    - pwd
    - virtualenv -p python3.7 venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages=htcondor
    - make test tests=src/toil/test/jobStores/jobStoreTest.py
    - make test tests=src/toil/test/sort/sortTest.py
    - make test tests=src/toil/test/provisioners/aws/awsProvisionerTest.py
    - make test tests=src/toil/test/provisioners/clusterScalerTest.py
#    - python -m pytest --duration=0 -s -r s src/toil/test/provisioners/gceProvisionerTest.py
# https://ucsc-ci.com/databiosphere/toil/-/jobs/38672
# guessing decorators are masking class as function?  ^  also, abstract class is run as normal test?  should hide.

py37_main:
  stage: basic_tests
  script:
    - pwd
    - virtualenv -p python3.7 venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages=htcondor
    - make test tests=src/toil/test/src
    - make test tests=src/toil/test/utils
#    - make test tests=src/toil/test/docs/scriptsTest.py::ToilDocumentationTest::testDocker

py37_appliance_build:
  stage: basic_tests
  script:
    - pwd
    - virtualenv -p python3.7 venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor awscli'
    # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
    - python setup_gitlab_docker.py
    - make push_docker

py37_integration:
  stage: integration
  script:
    - pwd
    - virtualenv -p python3.7 venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor awscli'
    - export TOIL_TEST_INTEGRATIVE=True
    - export TOIL_AWS_KEYNAME=id_rsa
    - export TOIL_AWS_ZONE=us-west-2a
    # This reads GITLAB_SECRET_FILE_SSH_KEYS
    - python setup_gitlab_ssh.py
    - chmod 400 /root/.ssh/id_rsa
    - make test tests=src/toil/test/jobStores/jobStoreTest.py

py37_provisioner_integration:
  stage: integration
  script:
    - pwd
    - virtualenv -p python3.7 venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor awscli'
    - python setup_gitlab_ssh.py && chmod 400 /root/.ssh/id_rsa
    - echo $'Host *\n    AddressFamily inet' > /root/.ssh/config
    - export LIBPROCESS_IP=127.0.0.1
    - python setup_gitlab_docker.py
    - export TOIL_TEST_INTEGRATIVE=True; export TOIL_AWS_KEYNAME=id_rsa; export TOIL_AWS_ZONE=us-west-2a
    # This reads GITLAB_SECRET_FILE_SSH_KEYS
    - python setup_gitlab_ssh.py
    - make test tests=src/toil/test/sort/sortTest.py
    - make test tests=src/toil/test/provisioners/clusterScalerTest.py
    - make test tests=src/toil/test/utils/utilsTest.py::UtilsTest::testAWSProvisionerUtils
    - make test tests=src/toil/test/provisioners/aws/awsProvisionerTest.py
    # - python -m pytest -s src/toil/test/provisioners/gceProvisionerTest.py  # needs env vars set to run

py37_google_jobstore:
  stage: integration
  script:
    - pwd
    - virtualenv -p python3.7 venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor awscli'
    - python setup_gitlab_ssh.py && chmod 400 /root/.ssh/id_rsa
    - echo $'Host *\n    AddressFamily inet' > /root/.ssh/config
    - export LIBPROCESS_IP=127.0.0.1
    - export TOIL_TEST_INTEGRATIVE=True
    - export GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_CREDENTIALS
    - export TOIL_GOOGLE_KEYNAME=id_rsa
    - export TOIL_GOOGLE_PROJECTID=toil-dev
    - make test tests=src/toil/test/jobStores/jobStoreTest.py::GoogleJobStoreTest

# Python3.8
py38_main:
  stage: basic_tests
  script:
    - pwd
    - virtualenv -p python3.8 venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor awscli'
    - make test tests=src/toil/test/src
    - make test tests=src/toil/test/utils
    - make test tests=src/toil/test/lib/test_ec2.py
    - make test tests=src/toil/test/lib/test_conversions.py

py38_appliance_build:
  stage: basic_tests
  script:
    - pwd
    - virtualenv -p python3.8 venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor awscli'
    # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
    - python setup_gitlab_docker.py
    - make push_docker

# Cactus-on-Kubernetes integration (as a script and not a pytest test)
py37_cactus_integration:
  stage: integration
  script:
    - set -e
    - virtualenv --system-site-packages --python python3.7 venv
    - . venv/bin/activate
    - pip install -U pip wheel
    - pip install .[aws,kubernetes]
    - export TOIL_KUBERNETES_OWNER=toiltest
    - export TOIL_AWS_SECRET_NAME=shared-s3-credentials
    - export TOIL_KUBERNETES_HOST_PATH=/data/scratch
    - export TOIL_WORKDIR=/var/lib/toil
    - export SINGULARITY_CACHEDIR=/var/lib/toil/singularity-cache
    - mkdir -p ${TOIL_WORKDIR}
    - BUCKET_NAME=toil-test-$RANDOM-$RANDOM-$RANDOM
    - cd
    - git clone https://github.com/ComparativeGenomicsToolkit/cactus.git --recursive
    - cd cactus
    - git fetch origin
    - git checkout a06d65042b7292de7f55e9396738ce69087f10af
    - git submodule update --init --recursive
    - pip install --upgrade setuptools pip
    - pip install --upgrade .
    - pip install psutil numpy --upgrade
    - toil clean aws:us-west-2:${BUCKET_NAME}
    - time cactus --setEnv SINGULARITY_DOCKER_HUB_MIRROR --batchSystem kubernetes --binariesMode singularity --clean always aws:us-west-2:${BUCKET_NAME} examples/evolverMammals.txt examples/evolverMammals.hal --root mr --defaultDisk "8G" --logDebug --disableCaching false