Fix test runner

.github/workflows/ci.yml

@@ -0,0 +1,20 @@
+name: CI
+
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+
+jobs:
+  build:
+    name: Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f # 2.3.4
+
+      - name: Run Tests in Docker
+        run: bin/run-tests-in-docker.sh

.gitignore

@@ -1,6 +1,8 @@
-test/output/
 .DS_Store
 **/.build
 **/Packages
 **/Package.resolved
 **/*.xcodeproj
+
+tests/*/results.json
+tests/*/.swiftpm/

Dockerfile

@@ -1,18 +1,13 @@
-FROM swift:latest AS builder
-# WORKDIR /opt/testrunner
+FROM swift:5.3.3-bionic AS builder
+
 COPY src/testrunner ./
 
-# Print Installed Swift Version
 RUN swift --version
-#RUN swift package clean
 RUN swift build --configuration release
 
-FROM swift:latest
+FROM swift:5.3.3-bionic
 WORKDIR /opt/test-runner/
 COPY bin/ bin/
 COPY --from=builder /.build/release/TestRunner bin/
 
-ENV NAME RUNALL
-
-ENTRYPOINT ["./bin/run.sh"]
-# ENTRYPOINT ["bin/TestRunner", "--help"]
+ENTRYPOINT ["/opt/test-runner/bin/run.sh"]

bin/run-in-docker.sh

@@ -1,39 +1,43 @@
-#!/usr/bin/env bash
-set -e
+#!/usr/bin/env sh
 
 # Synopsis:
-# Test runner for run.sh in a docker container
-# Takes the same arguments as run.sh (EXCEPT THAT SOLUTION AND OUTPUT PATH ARE RELATIVE)
-# Builds the Dockerfile
-# Runs the docker image passing along the initial arguments
+# Run the test runner on a solution using the test runner Docker image.
+# The test runner Docker image is built automatically.
 
 # Arguments:
 # $1: exercise slug
-# $2: **RELATIVE** path to solution folder (with trailing slash)
-# $3: **RELATIVE** path to output directory (with trailing slash)
+# $2: absolute path to solution folder
+# $3: absolute path to output directory
 
 # Output:
 # Writes the test results to a results.json file in the passed-in output directory.
-# The test results are formatted according to the specifications at https://github.com/exercism/automated-tests/blob/master/docs/interface.md
+# The test results are formatted according to the specifications at https://github.com/exercism/docs/blob/main/building/tooling/test-runners/interface.md
 
 # Example:
-# ./run-in-docker.sh two-fer ./relative/path/to/two-fer/solution/folder/ ./relative/path/to/output/directory/
+# ./bin/run-in-docker.sh two-fer /absolute/path/to/two-fer/solution/folder/ /absolute/path/to/output/directory/
 
-# If arguments not provided, print usage and exit
+# If any required arguments is missing, print the usage and exit
 if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then
-    echo "usage: run-in-docker.sh exercise-slug ./relative/path/to/solution/folder/ ./relative/path/to/output/directory/"
+    echo "usage: ./bin/run-in-docker.sh exercise-slug /absolute/path/to/solution/folder/ /absolute/path/to/output/directory/"
     exit 1
 fi
 
-# build docker image
-docker build --rm --no-cache -t swift-test-runner .
+slug="$1"
+input_dir="${2%/}"
+output_dir="${3%/}"
 
-# Create output directory if it doesn't exist
-output_dir="$3"
-mkdir -p "$output_dir"
+# Create the output directory if it doesn't exist
+mkdir -p "${output_dir}"
 
-# run image passing the arguments
+# Build the Docker image
+docker build --rm -t exercism/test-runner .
+
+# Run the Docker image using the settings mimicking the production environment
 docker run \
-    --mount type=bind,src=$PWD/$2,dst=/solution \
-    --mount type=bind,src=$PWD/$output_dir,dst=/output \
-    swift-test-runner $1 /solution/ /output/
+    --rm \
+    --network none \
+    --read-only \
+    --mount type=bind,src="${input_dir}",dst=/solution \
+    --mount type=bind,src="${output_dir}",dst=/output \
+    --mount type=tmpfs,dst=/tmp \
+    exercism/test-runner "${slug}" /solution /output 

bin/run-tests-in-docker.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env sh
+
+# Synopsis:
+# Test the test runner Docker image by running it against a predefined set of 
+# solutions with an expected output.
+# The test runner Docker image is built automatically.
+
+# Output:
+# Outputs the diff of the expected test results against the actual test results
+# generated by the test runner Docker image.
+
+# Example:
+# ./bin/run-tests-in-docker.sh
+
+# Build the Docker image
+docker build --rm -t exercism/test-runner .
+
+# Run the Docker image using the settings mimicking the production environment
+docker run \
+    --rm \
+    --network none \
+    --read-only \
+    --mount type=bind,src="${PWD}/tests",dst=/opt/test-runner/tests \
+    --mount type=tmpfs,dst=/tmp \
+    --workdir /opt/test-runner \
+    --entrypoint /opt/test-runner/bin/run-tests.sh \
+    exercism/test-runner

bin/run-tests.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env sh
+
+# Synopsis:
+# Test the test runner by running it against a predefined set of solutions 
+# with an expected output.
+
+# Output:
+# Outputs the diff of the expected test results against the actual test results
+# generated by the test runner.
+
+# Example:
+# ./bin/run-tests.sh
+
+exit_code=0
+
+# Iterate over all test directories
+for test_dir in tests/*; do
+    test_dir_name=$(basename "${test_dir}")
+    test_dir_path=$(realpath "${test_dir}")
+    results_file_path="${test_dir_path}/results.json"
+    expected_results_file_path="${test_dir_path}/expected_results.json"
+
+    bin/run.sh "${test_dir_name}" "${test_dir_path}" "${test_dir_path}"
+
+    echo "${test_dir_name}: comparing results.json to expected_results.json"
+    diff "${results_file_path}" "${expected_results_file_path}"
+
+    if [ $? -ne 0 ]; then
+        exit_code=1
+    fi
+done
+
+exit ${exit_code}

bin/run.sh

@@ -1,40 +1,36 @@
-#! /bin/sh
-set -e
+#!/usr/bin/env sh
 
-# If arguments not provided, print usage and exit
-if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then
-    echo "usage: run.sh exercise-slug ./relative/path/to/solution/folder/ ./relative/path/to/output/directory/"
-    exit 1
-fi
+# Synopsis:
+# Run the test runner on a solution.
+
+# Arguments:
+# $1: exercise slug
+# $2: absolute path to solution folder
+# $3: absolute path to output directory
 
-SLUG="$1"
-INPUT_DIR="$2"
-OUTPUT_DIR="$3"
+# Output:
+# Writes the test results to a results.json file in the passed-in output directory.
+# The test results are formatted according to the specifications at https://github.com/exercism/docs/blob/main/building/tooling/test-runners/interface.md
 
-BASEDIR=$(dirname "$0")
+# Example:
+# ./bin/run.sh two-fer /absolute/path/to/two-fer/solution/folder/ /absolute/path/to/output/directory/
 
-# echo "$SLUG: testing..."
-# echo "$1"
-# echo "$2"
-# echo "$3"
-# echo "-------------"
-RUNALL=true "${BASEDIR}"/TestRunner --slug "${SLUG}" --solution-directory "${INPUT_DIR}/${SLUG}" --output-directory "${OUTPUT_DIR}" --swift-location $(which swift) --build-directory "/tmp/"
+# If any required arguments is missing, print the usage and exit
+if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then
+    echo "usage: ./bin/run.sh exercise-slug /absolute/path/to/two-fer/solution/folder/ /absolute/path/to/output/directory/"
+    exit 1
+fi
 
-#echo "$SLUG: processing test output in $INPUT_DIR..."
-## PLACEHOLDER - OPTIONAL: Your language may support outputting results
-##   in the correct format
-#
-# Create $OUTPUT_DIR if it doesn't exist
-[ -d "$OUTPUT_DIR" ] || mkdir -p "$OUTPUT_DIR"
-#
-#echo "$SLUG: copying processed results to $OUTPUT_DIR..."
-## PLACEHOLDER - OPTIONAL: Your language may support placing results
-##   directly in $OUTPUT_DIR
-#cp "${INPUT_DIR}/results.json" "$OUTPUT_DIR"
+slug="$1"
+input_dir="${2%/}"
+output_dir="${3%/}"
 
-echo "$SLUG: comparing ${OUTPUT_DIR}/results"
-diff "${INPUT_DIR}/${SLUG}/results.json" "${OUTPUT_DIR}/results.json"
+# Create the output directory if it doesn't exist
+mkdir -p "${output_dir}"
 
-echo "$SLUG: OK\n-------------\n"
+echo "${slug}: testing..."
 
+# Run the tests for the provided implementation file
+RUNALL=true ./bin/TestRunner --slug "${slug}" --solution-directory "${input_dir}" --output-directory "${output_dir}" --swift-location $(which swift) --build-directory "/tmp"
 
+echo "${slug}: done"