From eed1e47a217043b2d632afc26a60c1c8074758c8 Mon Sep 17 00:00:00 2001 From: Jusong Yu Date: Tue, 17 Oct 2023 09:06:51 +0200 Subject: [PATCH] Restructure to build image first --- .github/actions/create-dev-env/action.yml | 35 +++++++++++++++ .github/actions/integration-tests/action.yml | 42 ++++++++++++++++++ .github/workflows/di-docker.yml | 45 ++++++++++++++++++++ .gitignore | 3 ++ docker/Dockerfile | 18 ++++++++ docker/docker-bake.hcl | 16 +++++++ docker/requirements-dev.txt | 11 +++++ tests_notebooks/conftest.py | 25 +++++------ tests_notebooks/docker-compose.yml | 9 ++-- 9 files changed, 183 insertions(+), 21 deletions(-) create mode 100644 .github/actions/create-dev-env/action.yml create mode 100644 .github/actions/integration-tests/action.yml create mode 100644 .github/workflows/di-docker.yml create mode 100644 docker/Dockerfile create mode 100644 docker/docker-bake.hcl create mode 100644 docker/requirements-dev.txt diff --git a/.github/actions/create-dev-env/action.yml b/.github/actions/create-dev-env/action.yml new file mode 100644 index 000000000..934eabeac --- /dev/null +++ b/.github/actions/create-dev-env/action.yml @@ -0,0 +1,35 @@ +--- +name: Build environment +description: Create build environment + +inputs: + architecture: + description: architecture to be run on + required: true + type: string + +runs: + using: composite + steps: + # actions/setup-python doesn't support Linux arm64 runners + # See: https://github.com/actions/setup-python/issues/108 + # python3 is manually preinstalled in the arm64 VM self-hosted runner + - name: Set Up Python 🐍 + if: ${{ inputs.architecture == 'amd64' }} + uses: actions/setup-python@v4 + with: + python-version: 3.x + + - name: Install Dev Dependencies 📦 + if: ${{ inputs.architecture == 'amd64' }} + run: | + pip install --upgrade pip + pip install --upgrade -r docker/requirements-dev.txt + shell: bash + + - name: Install Dev Dependencies 📦 + if: ${{ inputs.architecture == 'arm64' }} + run: | + pip install --upgrade pip + pip install --upgrade -r docker/requirements-dev-arm64.txt + shell: bash diff --git a/.github/actions/integration-tests/action.yml b/.github/actions/integration-tests/action.yml new file mode 100644 index 000000000..1969afc48 --- /dev/null +++ b/.github/actions/integration-tests/action.yml @@ -0,0 +1,42 @@ +--- +name: Downstream tests +description: Integration downstream tests the bulid image + +runs: + using: composite + + steps: + - name: Set jupyter token env + run: echo "JUPYTER_TOKEN=$(openssl rand -hex 32)" >> $GITHUB_ENV + shell: bash + + - name: Run pytest to test image is working + run: TAG=newly-baked pytest tests_integration/test_image.py + shell: bash + + # The Firefox and its engine geckodrive need do be installed manually to run + - name: Install Firefox + uses: browser-actions/setup-firefox@latest + with: + firefox-version: '96.0' + + - name: Install geckodriver + run: | + wget -c https://github.com/mozilla/geckodriver/releases/download/v0.30.0/geckodriver-v0.30.0-linux64.tar.gz + tar xf geckodriver-v0.30.0-linux64.tar.gz -C /usr/local/bin + shell: bash + + - name: Run pytest for firefox + run: TAG=newly-baked pytest --driver Firefox tests_integration/test_app.py + shell: bash + + - name: Run pytest for Chrome + run: TAG=newly-baked pytest --driver Chrome tests_integration/test_app.py + shell: bash + + - name: Upload screenshots as artifacts + uses: actions/upload-artifact@v3 + with: + name: Screenshots-CI + path: screenshots/ + if-no-files-found: error diff --git a/.github/workflows/di-docker.yml b/.github/workflows/di-docker.yml new file mode 100644 index 000000000..9f34817da --- /dev/null +++ b/.github/workflows/di-docker.yml @@ -0,0 +1,45 @@ +--- +# Run basic tests for this app on the latest aiidalab-docker image. +name: smoke tests on notebooks + +on: [push, pull_request] + + +# https://docs.github.com/en/actions/using-jobs/using-concurrency +concurrency: + # only cancel in-progress jobs or runs for the current workflow - matches against branch & tags + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + + build-test: + + runs-on: ubuntu-latest + continue-on-error: true + + steps: + - name: Checkout Repo ⚡️ + uses: actions/checkout@v3 + - name: Set Up Python 🐍 + uses: actions/setup-python@v4 + with: + python-version: 3.x + + - name: Install Dev Dependencies 📦 + run: | + pip install --upgrade pip + pip install --upgrade -r docker/requirements-dev.txt + + - name: Build image 🛠 + working-directory: docker + run: docker buildx bake -f docker-bake.hcl --load + env: + # Use buildx + DOCKER_BUILDKIT: 1 + # Full logs for CI build + BUILDKIT_PROGRESS: plain + shell: bash + + - name: Run tests ✅ + uses: ./.github/actions/integration-tests diff --git a/.gitignore b/.gitignore index 66e045e5e..f6589f338 100644 --- a/.gitignore +++ b/.gitignore @@ -109,3 +109,6 @@ venv.bak/ .DS_Store .vscode + +# screenshots +screenshots/ diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 000000000..a0f9fbb1c --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,18 @@ +# syntax=docker/dockerfile:1 +FROM aiidalab/full-stack:latest + +# Copy whole repo and pre-install the dependencies and app to the tmp folder. +# In the before notebook scripts the app will be re-installed by moving it to the app folder. +ENV PREINSTALL_APP_FOLDER ${HOME}/aiidalab-widgets-base +COPY --chown=${NB_UID}:${NB_GID} --from=src . ${PREINSTALL_APP_FOLDER} + +USER ${NB_USER} + +RUN cd ${PREINSTALL_APP_FOLDER} && \ + # Remove all untracked files and directories. For example the setup lock flag file. + git clean -fx && \ + pip install . --no-cache-dir && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +WORKDIR "/home/${NB_USER}" diff --git a/docker/docker-bake.hcl b/docker/docker-bake.hcl new file mode 100644 index 000000000..d21d3363e --- /dev/null +++ b/docker/docker-bake.hcl @@ -0,0 +1,16 @@ +# docker-bake.hcl for building QeApp images +group "default" { + targets = ["awb"] +} + +variable "ORGANIZATION" { + default = "aiidalab" +} + +target "awb" { + tags = ["${ORGANIZATION}/aiidalab-widgets-base:newly-baked"] + context = "." + contexts = { + src = ".." + } +} diff --git a/docker/requirements-dev.txt b/docker/requirements-dev.txt new file mode 100644 index 000000000..4c8d31491 --- /dev/null +++ b/docker/requirements-dev.txt @@ -0,0 +1,11 @@ +docker +requests +pre-commit +pytest +pytest-docker + +# test dependencies +pytest-selenium +pytest-html<4.0 +selenium~=4.9.0 +webdriver-manager diff --git a/tests_notebooks/conftest.py b/tests_notebooks/conftest.py index 17bf6c10b..4f1099ed5 100644 --- a/tests_notebooks/conftest.py +++ b/tests_notebooks/conftest.py @@ -41,12 +41,13 @@ def docker_compose(docker_services): @pytest.fixture(scope="session") def aiidalab_exec(docker_compose): - def execute(command, user=None, **kwargs): - workdir = "/home/jovyan/apps/aiidalab-widgets-base" + def execute(command, user=None, workdir=None, **kwargs): + opts = "-T" if user: - command = f"exec --workdir {workdir} -T --user={user} aiidalab {command}" - else: - command = f"exec --workdir {workdir} -T aiidalab {command}" + opts = f"{opts} --user={user}" + if workdir: + opts = f"{opts} --workdir={workdir}" + command = f"exec {opts} aiidalab {command}" return docker_compose.execute(command, **kwargs) @@ -56,17 +57,10 @@ def execute(command, user=None, **kwargs): @pytest.fixture(scope="session", autouse=True) def notebook_service(docker_ip, docker_services, aiidalab_exec): """Ensure that HTTP service is up and responsive.""" - # Directory ~/apps/aiidalab-widgets-base/ is mounted by docker, - # make it writeable for jovyan user, needed for `pip install` - aiidalab_exec("chmod -R a+rw /home/jovyan/apps/aiidalab-widgets-base", user="root") - - # Install AWB with extra dependencies for SmilesWidget - aiidalab_exec("pip install -U .[smiles]") - # `port_for` takes a container port and returns the corresponding host port port = docker_services.port_for("aiidalab", 8888) url = f"http://{docker_ip}:{port}" - token = os.environ["JUPYTER_TOKEN"] + token = os.environ.get("JUPYTER_TOKEN", "aiidalab") docker_services.wait_until_responsive( timeout=30.0, pause=0.1, check=lambda: is_responsive(url) ) @@ -78,7 +72,7 @@ def selenium_driver(selenium, notebook_service): def _selenium_driver(nb_path): url, token = notebook_service url_with_token = urljoin( - url, f"apps/apps/aiidalab-widgets-base/{nb_path}?token={token}" + url, f"apps/aiidalab-widgets-base/{nb_path}?token={token}" ) selenium.get(f"{url_with_token}") # By default, let's allow selenium functions to retry for 10s @@ -106,7 +100,8 @@ def final_screenshot(request, screenshot_dir, selenium): Screenshot name is generated from the test function name by stripping the 'test_' prefix """ - screenshot_name = f"{request.function.__name__[5:]}.png" + browser_name = selenium.capabilities["browserName"] + screenshot_name = f"{request.function.__name__[5:]}-{browser_name}.png" screenshot_path = Path.joinpath(screenshot_dir, screenshot_name) yield selenium.get_screenshot_as_file(screenshot_path) diff --git a/tests_notebooks/docker-compose.yml b/tests_notebooks/docker-compose.yml index 3beae4897..c445968ff 100644 --- a/tests_notebooks/docker-compose.yml +++ b/tests_notebooks/docker-compose.yml @@ -4,15 +4,12 @@ version: '3.4' services: aiidalab: - image: aiidalab/full-stack:${TAG:-latest} + image: ${REGISTRY:-}${QE_IMAGE:-aiidalab/aiidalab-widgets-base}:${TAG:-newly-baked} environment: - RMQHOST: messaging TZ: Europe/Zurich DOCKER_STACKS_JUPYTER_CMD: notebook SETUP_DEFAULT_AIIDA_PROFILE: 'true' AIIDALAB_DEFAULT_APPS: '' - JUPYTER_TOKEN: ${JUPYTER_TOKEN} - volumes: - - ..:/home/jovyan/apps/aiidalab-widgets-base + JUPYTER_TOKEN: ${JUPYTER_TOKEN:-aiidalab} ports: - - 8998:8888 + - 0.0.0.0:${AIIDALAB_PORT:-8998}:8888