Streamline e2e commands (#803)

## Ticket

Resolves #795 

## Changes

- Fix E2E ui mode with new npm `e2e-test:ui` command
- Add `e2e-merge-reports` npm run command
- Add `ENTRYPOINT` with optional additinoal `CMD` args in
`./e2e/Dockerfile`
- allow `E2E_ARGS` to be passed via make native targets
- Make `BASE_URL` optional in `./e2e/run-e2e-tests`
- Allow command args to be passed to `./e2e/run-e2e-tests`
- Update several `e2e-clean` make targets
- Use variable for e2e image name in Makefile
- Update e2e docs

Makefile

@@ -28,14 +28,16 @@ __check_defined = \
 
 .PHONY : \
 	e2e-build \
+	e2e-clean \
+	e2e-clean-image \
 	e2e-clean-report \
-	e2e-delete-image \
 	e2e-merge-reports \
 	e2e-setup-ci \
 	e2e-setup-native \
 	e2e-show-report \
 	e2e-test \
 	e2e-test-native \
+	e2e-test-native-ui \
 	help \
 	infra-check-app-database-roles \
 	infra-check-compliance-checkov \
@@ -74,51 +76,63 @@ __check_defined = \
 ## End-to-end (E2E) Testing ##
 ##############################
 
+# Include project name in image name so that image name
+# does not conflict with other images during local development.
+# The e2e test image includes the test suite for all apps and therefore isn't specific to each app.
+E2E_IMAGE_NAME := $(PROJECT_ROOT)-e2e
+
 e2e-build: ## Build the e2e Docker image, if not already built, using ./e2e/Dockerfile
-	docker build -t playwright-e2e -f ./e2e/Dockerfile .
+	docker build -t $(E2E_IMAGE_NAME) -f ./e2e/Dockerfile .
+
+e2e-clean: ## Clean both the e2e reports and e2e Docker image
+e2e-clean: e2e-clean-report e2e-clean-image
+
+e2e-clean-image: ## Clean the Docker image for e2e tests
+	docker rmi -f $(E2E_IMAGE_NAME) 2>/dev/null || echo "Docker image $(E2E_IMAGE_NAME) does not exist, skipping."
 
-e2e-clean-report: ## Remove the local ./e2e/playwright-report and ./e2e/test-results folder and their contents
+e2e-clean-report: ## Remove the local e2e report folders and content
 	rm -rf ./e2e/playwright-report
 	rm -rf ./e2e/blob-report
 	rm -rf ./e2e/test-results
 
-e2e-delete-image: ## Delete the Docker image for e2e tests
-	@docker rmi -f playwright-e2e 2>/dev/null || echo "Docker image playwright-e2e does not exist, skipping."
-
-e2e-merge-reports: ## Merge Playwright blob reports from multiple shards into an HTML report
-	@cd e2e && npx playwright merge-reports --reporter html blob-report
+e2e-merge-reports: ## Merge E2E blob reports from multiple shards into an HTML report
+	cd e2e && npm run e2e-merge-reports
 
 e2e-setup-ci: ## Setup end-to-end tests for CI
-	@cd e2e && npm ci
-	@cd e2e && npx playwright install --with-deps
+	cd e2e && npm run e2e-setup
 
 e2e-setup-native: ## Setup end-to-end tests
-	@cd e2e && npm install
-	@cd e2e && npx playwright install --with-deps
+	cd e2e && npm install
+	cd e2e && npm run e2e-setup
 
-e2e-show-report: ## Show the ./e2e/playwright-report
-	@cd e2e && npx playwright show-report
+e2e-show-report: ## Show the E2E report
+	cd e2e && npm run e2e-show-report
 
-e2e-test: ## Run E2E Playwright tests in a Docker container and copy the report locally
+e2e-test: ## Run E2E tests in a Docker container and copy the report locally
 e2e-test: e2e-build
 	@:$(call check_defined, APP_NAME, You must pass in a specific APP_NAME)
 	@:$(call check_defined, BASE_URL, You must pass in a BASE_URL)
 	docker run --rm\
-		--name playwright-e2e-container \
+		--name $(E2E_IMAGE_NAME)-container \
 		-e APP_NAME=$(APP_NAME) \
 		-e BASE_URL=$(BASE_URL) \
 		-e CURRENT_SHARD=$(CURRENT_SHARD) \
 		-e TOTAL_SHARDS=$(TOTAL_SHARDS) \
 		-e CI=$(CI) \
 		-v $(PWD)/e2e/playwright-report:/e2e/playwright-report \
 		-v $(PWD)/e2e/blob-report:/e2e/blob-report \
-		playwright-e2e
+		$(E2E_IMAGE_NAME) \
+		$(E2E_ARGS)
 
-e2e-test-native: ## Run end-to-end tests
+e2e-test-native: ## Run end-to-end tests natively
 	@:$(call check_defined, APP_NAME, You must pass in a specific APP_NAME)
-	@:$(call check_defined, BASE_URL, You must pass in a BASE_URL)
 	@echo "Running e2e tests with CI=${CI}, APP_NAME=${APP_NAME}, BASE_URL=${BASE_URL}"
-	@cd e2e/$(APP_NAME) && APP_NAME=$(APP_NAME) BASE_URL=$(BASE_URL) npx playwright test $(E2E_ARGS)
+	cd e2e && APP_NAME=$(APP_NAME) BASE_URL=$(BASE_URL) npm run e2e-test -- $(E2E_ARGS)
+
+e2e-test-native-ui: ## Run end-to-end tests natively in UI mode
+	@:$(call check_defined, APP_NAME, You must pass in a specific APP_NAME)
+	@echo "Running e2e UI tests natively with APP_NAME=$(APP_NAME), BASE_URL=$(BASE_URL)"
+	cd e2e && APP_NAME=$(APP_NAME) BASE_URL=$(BASE_URL) npm run e2e-test:ui -- $(E2E_ARGS)
 
 ###########
 ## Infra ##

docs/e2e/e2e-checks.md

@@ -4,7 +4,7 @@
 
 This repository uses [Playwright](https://playwright.dev/) to perform end-to-end (E2E) tests. The tests can be run locally (natively or within Docker), but they also run on [Pull Request preview environments](../infra/pull-request-environments.md). This ensures that any new code changes are validated through E2E tests before being merged.
 
-By default, tests are sharded across 3 concurrent runs to reduce total runtime. As the test suite grows, consider increasing the shard count to further optimize execution time. This is set in the [workflow file](../../.github/workflows/e2e-tests.yml#L22).
+By default in CI, tests are sharded across 3 concurrent runs to reduce total runtime. As the test suite grows, consider increasing the shard count to further optimize execution time. This is set in the [workflow file](../../.github/workflows/e2e-tests.yml#L22).
 
 ## Folder Structure
 In order to support e2e for multiple apps, the folder structure will include a base playwright config (`./e2e/playwright.config.js`), and app-specific derived playwright config that override the base config. See the example folder structure below:
@@ -22,10 +22,10 @@ In order to support e2e for multiple apps, the folder structure will include a b
 ```
 
 Some highlights:
-- By default, the base config is defined to run on a minimal browser-set (desktop and mobile chrome). Browsers can be added in the app-specific playwright config.
-- Snapshots will be output locally (in the `./e2e` folder or the container) - or in the artifacts of the CI job
-- HTML reports are output to the `playwright-report` folder
-- Accessibility testing can be performed using the `@axe-core/playwright` package (https://playwright.dev/docs/accessibility-testing)
+>- By default, the base config is defined to run on a minimal browser-set (desktop and mobile chrome). Browsers can be added in the app-specific playwright config.
+>- Snapshots will be output locally (in the `./e2e` folder or the container) - or in the artifacts of the CI job
+>- HTML reports are output to the `playwright-report` folder
+>- Accessibility testing can be performed using the `@axe-core/playwright` package (https://playwright.dev/docs/accessibility-testing)
 
 ## Run tests locally
 
@@ -38,7 +38,7 @@ Then, run end-to-end tests using Docker with:
 make e2e-test APP_NAME=app BASE_URL=http://host.docker.internal:3000
 ```
 
-*Note that `BASE_URL` cannot be `localhost`
+>*Note that `BASE_URL` cannot be `localhost`
 
 
 ### Run tests natively
@@ -53,10 +53,20 @@ make e2e-setup-native
 
 Then, run the tests with your app name and base url:
 ```bash
-make e2e-test-native APP_NAME=app BASE_URL=http://localhost:3000
+make e2e-test-native APP_NAME=app
 ```
+>* `BASE_URL` is optional for both `e2e-test-native` and `e2e-test-native-ui` targets. It will by default use the [app-specific](../../e2e/app/playwright.config.js) `baseURL`
 
-#### Run tests in parallel 
+### Run tests in UI mode
+
+When developing or debugging tests, it’s often helpful to see them running in real-time. You can achieve this by running the e2e tests in UI mode:
+
+```
+make e2e-test-native-ui APP_NAME=app
+```
+
+
+#### Run tests in parallel
 
 The following commands split test execution into 3 separate shards, with results consolidated into a merged report located in `./e2e/blob-report`. This setup emulates how the sharded tests run in CI.
 ```
@@ -86,7 +96,7 @@ To clean the report folder you can run:
 make e2e-clean-report
 ```
 
-*On CI, the report shows up in the Github Actions artifacts tab
+>*On CI, the report shows up in the Github Actions artifacts tab
 
 
 ### PR preview environments

e2e/Dockerfile

@@ -2,19 +2,22 @@
 FROM mcr.microsoft.com/playwright:v1.48.1-noble
 
 # Install make and any other necessary dependencies
-RUN apt-get update && apt-get install -y make
+RUN apt-get update
 
 # Set the working directory to the root of the project inside the Docker container
-WORKDIR /
+WORKDIR /e2e
 
 # Setup npm install layer that can be cached
-COPY Makefile /Makefile
-COPY e2e/package.json e2e/package-lock.json /e2e/
+COPY e2e/package.json e2e/package-lock.json ./
 
 # install deps
-RUN make e2e-setup-ci
+RUN npm ci
+RUN npm run e2e-setup
 
 # Copy entire e2e folder over
 COPY e2e /e2e
 
-CMD ["make", "e2e-test-native"]
+ENTRYPOINT ["npm", "run", "e2e-test", "--"]
+
+# Optional additional args
+CMD [""]

e2e/app/playwright.config.js

@@ -6,7 +6,7 @@ export default defineConfig(deepMerge(
   baseConfig,
   {
     use: {
-      baseURL: baseConfig.use.baseURL || "localhost:3000"
+      baseURL: baseConfig.use.baseURL || "http://localhost:3000"
     },
   }
 ));

e2e/package.json

@@ -2,9 +2,11 @@
     "name": "e2e",
     "version": "1.0.0",
     "scripts": {
+      "e2e-merge-reports": "npx playwright merge-reports --reporter html blob-report",
       "e2e-setup": "npx playwright install --with-deps",
+      "e2e-show-report": "npx playwright show-report",
       "e2e-test": "./run-e2e-test",
-      "e2e-test:ui": "npx playwright test --ui"
+      "e2e-test:ui": "./run-e2e-test --ui"
     },
     "devDependencies": {
       "@playwright/test": "^1.45.1",

e2e/run-e2e-test

@@ -1,13 +1,17 @@
 #!/bin/bash
 #
 # Script to run Playwright tests with a specified app name.
-# Requires the APP_NAME environment variable to be set.
+# APP_NAME is required to be set, while BASE_URL is optional.
 
 # Ensure APP_NAME is provided
 if [[ -z "${APP_NAME}" ]]; then
   echo "You must pass in a specific APP_NAME. IE: APP_NAME=app npm run e2e-test" >&2
   exit 1
 fi
 
-# Run Playwright tests with the specified app name.
-npx playwright test --config "${APP_NAME}/playwright.config.js"
+# Inform about optional BASE_URL usage
+if [[ -z "${BASE_URL}" ]]; then
+  echo "BASE_URL not specified. Using default BASE_URL specified in playwright.config.js"
+fi
+
+npx playwright test --config "${APP_NAME}/playwright.config.js" "$@"