diff --git a/Makefile b/Makefile index 722e8c01..6f571694 100644 --- a/Makefile +++ b/Makefile @@ -28,14 +28,16 @@ __check_defined = \ .PHONY : \ e2e-build \ + e2e-clean \ + e2e-clean-image \ e2e-clean-report \ - e2e-delete-image \ e2e-merge-reports \ e2e-setup-ci \ e2e-setup-native \ e2e-show-report \ e2e-test \ e2e-test-native \ + e2e-test-native-ui \ help \ infra-check-app-database-roles \ infra-check-compliance-checkov \ @@ -74,37 +76,44 @@ __check_defined = \ ## End-to-end (E2E) Testing ## ############################## +# Include project name in image name so that image name +# does not conflict with other images during local development. +# The e2e test image includes the test suite for all apps and therefore isn't specific to each app. +E2E_IMAGE_NAME := $(PROJECT_ROOT)-e2e + e2e-build: ## Build the e2e Docker image, if not already built, using ./e2e/Dockerfile - docker build -t playwright-e2e -f ./e2e/Dockerfile . + docker build -t $(E2E_IMAGE_NAME) -f ./e2e/Dockerfile . + +e2e-clean: ## Clean both the e2e reports and e2e Docker image +e2e-clean: e2e-clean-report e2e-clean-image + +e2e-clean-image: ## Clean the Docker image for e2e tests + docker rmi -f $(E2E_IMAGE_NAME) 2>/dev/null || echo "Docker image $(E2E_IMAGE_NAME) does not exist, skipping." -e2e-clean-report: ## Remove the local ./e2e/playwright-report and ./e2e/test-results folder and their contents +e2e-clean-report: ## Remove the local e2e report folders and content rm -rf ./e2e/playwright-report rm -rf ./e2e/blob-report rm -rf ./e2e/test-results -e2e-delete-image: ## Delete the Docker image for e2e tests - @docker rmi -f playwright-e2e 2>/dev/null || echo "Docker image playwright-e2e does not exist, skipping." - -e2e-merge-reports: ## Merge Playwright blob reports from multiple shards into an HTML report - @cd e2e && npx playwright merge-reports --reporter html blob-report +e2e-merge-reports: ## Merge E2E blob reports from multiple shards into an HTML report + cd e2e && npm run e2e-merge-reports e2e-setup-ci: ## Setup end-to-end tests for CI - @cd e2e && npm ci - @cd e2e && npx playwright install --with-deps + cd e2e && npm run e2e-setup e2e-setup-native: ## Setup end-to-end tests - @cd e2e && npm install - @cd e2e && npx playwright install --with-deps + cd e2e && npm install + cd e2e && npm run e2e-setup -e2e-show-report: ## Show the ./e2e/playwright-report - @cd e2e && npx playwright show-report +e2e-show-report: ## Show the E2E report + cd e2e && npm run e2e-show-report -e2e-test: ## Run E2E Playwright tests in a Docker container and copy the report locally +e2e-test: ## Run E2E tests in a Docker container and copy the report locally e2e-test: e2e-build @:$(call check_defined, APP_NAME, You must pass in a specific APP_NAME) @:$(call check_defined, BASE_URL, You must pass in a BASE_URL) docker run --rm\ - --name playwright-e2e-container \ + --name $(E2E_IMAGE_NAME)-container \ -e APP_NAME=$(APP_NAME) \ -e BASE_URL=$(BASE_URL) \ -e CURRENT_SHARD=$(CURRENT_SHARD) \ @@ -112,13 +121,18 @@ e2e-test: e2e-build -e CI=$(CI) \ -v $(PWD)/e2e/playwright-report:/e2e/playwright-report \ -v $(PWD)/e2e/blob-report:/e2e/blob-report \ - playwright-e2e + $(E2E_IMAGE_NAME) \ + $(E2E_ARGS) -e2e-test-native: ## Run end-to-end tests +e2e-test-native: ## Run end-to-end tests natively @:$(call check_defined, APP_NAME, You must pass in a specific APP_NAME) - @:$(call check_defined, BASE_URL, You must pass in a BASE_URL) @echo "Running e2e tests with CI=${CI}, APP_NAME=${APP_NAME}, BASE_URL=${BASE_URL}" - @cd e2e/$(APP_NAME) && APP_NAME=$(APP_NAME) BASE_URL=$(BASE_URL) npx playwright test $(E2E_ARGS) + cd e2e && APP_NAME=$(APP_NAME) BASE_URL=$(BASE_URL) npm run e2e-test -- $(E2E_ARGS) + +e2e-test-native-ui: ## Run end-to-end tests natively in UI mode + @:$(call check_defined, APP_NAME, You must pass in a specific APP_NAME) + @echo "Running e2e UI tests natively with APP_NAME=$(APP_NAME), BASE_URL=$(BASE_URL)" + cd e2e && APP_NAME=$(APP_NAME) BASE_URL=$(BASE_URL) npm run e2e-test:ui -- $(E2E_ARGS) ########### ## Infra ## diff --git a/docs/e2e/e2e-checks.md b/docs/e2e/e2e-checks.md index ac2b53f6..a8c9e4c1 100644 --- a/docs/e2e/e2e-checks.md +++ b/docs/e2e/e2e-checks.md @@ -4,7 +4,7 @@ This repository uses [Playwright](https://playwright.dev/) to perform end-to-end (E2E) tests. The tests can be run locally (natively or within Docker), but they also run on [Pull Request preview environments](../infra/pull-request-environments.md). This ensures that any new code changes are validated through E2E tests before being merged. -By default, tests are sharded across 3 concurrent runs to reduce total runtime. As the test suite grows, consider increasing the shard count to further optimize execution time. This is set in the [workflow file](../../.github/workflows/e2e-tests.yml#L22). +By default in CI, tests are sharded across 3 concurrent runs to reduce total runtime. As the test suite grows, consider increasing the shard count to further optimize execution time. This is set in the [workflow file](../../.github/workflows/e2e-tests.yml#L22). ## Folder Structure In order to support e2e for multiple apps, the folder structure will include a base playwright config (`./e2e/playwright.config.js`), and app-specific derived playwright config that override the base config. See the example folder structure below: @@ -22,10 +22,10 @@ In order to support e2e for multiple apps, the folder structure will include a b ``` Some highlights: -- By default, the base config is defined to run on a minimal browser-set (desktop and mobile chrome). Browsers can be added in the app-specific playwright config. -- Snapshots will be output locally (in the `./e2e` folder or the container) - or in the artifacts of the CI job -- HTML reports are output to the `playwright-report` folder -- Accessibility testing can be performed using the `@axe-core/playwright` package (https://playwright.dev/docs/accessibility-testing) +>- By default, the base config is defined to run on a minimal browser-set (desktop and mobile chrome). Browsers can be added in the app-specific playwright config. +>- Snapshots will be output locally (in the `./e2e` folder or the container) - or in the artifacts of the CI job +>- HTML reports are output to the `playwright-report` folder +>- Accessibility testing can be performed using the `@axe-core/playwright` package (https://playwright.dev/docs/accessibility-testing) ## Run tests locally @@ -38,7 +38,7 @@ Then, run end-to-end tests using Docker with: make e2e-test APP_NAME=app BASE_URL=http://host.docker.internal:3000 ``` -*Note that `BASE_URL` cannot be `localhost` +>*Note that `BASE_URL` cannot be `localhost` ### Run tests natively @@ -53,10 +53,20 @@ make e2e-setup-native Then, run the tests with your app name and base url: ```bash -make e2e-test-native APP_NAME=app BASE_URL=http://localhost:3000 +make e2e-test-native APP_NAME=app ``` +>* `BASE_URL` is optional for both `e2e-test-native` and `e2e-test-native-ui` targets. It will by default use the [app-specific](../../e2e/app/playwright.config.js) `baseURL` -#### Run tests in parallel +### Run tests in UI mode + +When developing or debugging tests, it’s often helpful to see them running in real-time. You can achieve this by running the e2e tests in UI mode: + +``` +make e2e-test-native-ui APP_NAME=app +``` + + +#### Run tests in parallel The following commands split test execution into 3 separate shards, with results consolidated into a merged report located in `./e2e/blob-report`. This setup emulates how the sharded tests run in CI. ``` @@ -86,7 +96,7 @@ To clean the report folder you can run: make e2e-clean-report ``` -*On CI, the report shows up in the Github Actions artifacts tab +>*On CI, the report shows up in the Github Actions artifacts tab ### PR preview environments diff --git a/e2e/Dockerfile b/e2e/Dockerfile index 1fd1b7e5..6a2c3d72 100644 --- a/e2e/Dockerfile +++ b/e2e/Dockerfile @@ -2,19 +2,22 @@ FROM mcr.microsoft.com/playwright:v1.48.1-noble # Install make and any other necessary dependencies -RUN apt-get update && apt-get install -y make +RUN apt-get update # Set the working directory to the root of the project inside the Docker container -WORKDIR / +WORKDIR /e2e # Setup npm install layer that can be cached -COPY Makefile /Makefile -COPY e2e/package.json e2e/package-lock.json /e2e/ +COPY e2e/package.json e2e/package-lock.json ./ # install deps -RUN make e2e-setup-ci +RUN npm ci +RUN npm run e2e-setup # Copy entire e2e folder over COPY e2e /e2e -CMD ["make", "e2e-test-native"] +ENTRYPOINT ["npm", "run", "e2e-test", "--"] + +# Optional additional args +CMD [""] diff --git a/e2e/app/playwright.config.js b/e2e/app/playwright.config.js index 77a72fca..6621e75d 100644 --- a/e2e/app/playwright.config.js +++ b/e2e/app/playwright.config.js @@ -6,7 +6,7 @@ export default defineConfig(deepMerge( baseConfig, { use: { - baseURL: baseConfig.use.baseURL || "localhost:3000" + baseURL: baseConfig.use.baseURL || "http://localhost:3000" }, } )); diff --git a/e2e/package.json b/e2e/package.json index 5de9287d..71c26fde 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -2,9 +2,11 @@ "name": "e2e", "version": "1.0.0", "scripts": { + "e2e-merge-reports": "npx playwright merge-reports --reporter html blob-report", "e2e-setup": "npx playwright install --with-deps", + "e2e-show-report": "npx playwright show-report", "e2e-test": "./run-e2e-test", - "e2e-test:ui": "npx playwright test --ui" + "e2e-test:ui": "./run-e2e-test --ui" }, "devDependencies": { "@playwright/test": "^1.45.1", diff --git a/e2e/run-e2e-test b/e2e/run-e2e-test index 7aedf547..f46ee39e 100755 --- a/e2e/run-e2e-test +++ b/e2e/run-e2e-test @@ -1,7 +1,7 @@ #!/bin/bash # # Script to run Playwright tests with a specified app name. -# Requires the APP_NAME environment variable to be set. +# APP_NAME is required to be set, while BASE_URL is optional. # Ensure APP_NAME is provided if [[ -z "${APP_NAME}" ]]; then @@ -9,5 +9,9 @@ if [[ -z "${APP_NAME}" ]]; then exit 1 fi -# Run Playwright tests with the specified app name. -npx playwright test --config "${APP_NAME}/playwright.config.js" +# Inform about optional BASE_URL usage +if [[ -z "${BASE_URL}" ]]; then + echo "BASE_URL not specified. Using default BASE_URL specified in playwright.config.js" +fi + +npx playwright test --config "${APP_NAME}/playwright.config.js" "$@"