test(cat-gateway): schemathesis for health endpoints (#1485)

* wip * wip * wip * wip * wip * wip * fix: remove scylla earthfile * feat: runs schemathesis only on api/started endpoint * clean up * fix: blueprint * fix: blueprint * fix: blueprint * fix: blueprint * fix: remove hooks * fix: remove some checks * feat: adds seed param to chemathesis package * fix: poetry * fix: earthly dockerd bug * fix: remove checks all param * fix: remove duplicated function * fix: regex * adds all check * chore: clean up * clean up * chore: update schemathesis version * fix: update schemathesis option to latest version * fixes * fix: st target failing statment --------- Co-authored-by: Steven Johnson <[email protected]> Co-authored-by: Oleksandr Prokhorenko <[email protected]>
input-output-hk · Jan 13, 2025 · a40efb5 · a40efb5
1 parent 40afc42
commit a40efb5
Show file tree

Hide file tree

Showing 4 changed files with 23 additions and 11 deletions.
diff --git a/.gitignore b/.gitignore
@@ -123,4 +123,8 @@ test-results/
 /playwright/.cache/
 extensions/
 browser_extensions/
-!catalyst_voices/packages/internal/catalyst_voices_driver/lib/src/extensions/
+!catalyst_voices/packages/internal/catalyst_voices_driver/lib/src/extensions/
+
+#Test report and coverage
+*.junit-report.xml
+*.coverage.info
diff --git a/Overall, b/Overall,
diff --git a/catalyst-gateway/tests/schemathesis_tests/.gitignore b/catalyst-gateway/tests/schemathesis_tests/.gitignore
@@ -1,3 +1,3 @@
 __pycache__
-junit-report.xml
-.hypothesis/
+.hypothesis/
+cassette.yaml
diff --git a/catalyst-gateway/tests/schemathesis_tests/Earthfile b/catalyst-gateway/tests/schemathesis_tests/Earthfile
@@ -3,21 +3,25 @@ VERSION 0.8
 package-schemathesis:
     FROM python:3.12-alpine3.20
     # TODO: https://github.com/input-output-hk/catalyst-voices/issues/465
+    ARG openapi_spec
+    # optional argument that can be used to pass a --hypothesis-seed to replicate specific test failures
+    ARG seed
+    ARG version=3.39.5
+
     RUN apk add --no-cache gcc musl-dev
-    RUN python -m pip install schemathesis==3.27.1
+    RUN python -m pip install schemathesis==$version
     RUN mkdir /results
     COPY ./hooks/hooks.py .
     VOLUME /results
-    ARG openapi_spec
-    # optional argument that can be used to pass a --hypothesis-seed to replicate specific test failures
-    ARG seed
-    ENTRYPOINT st run --endpoint '^/api/v1/health' $openapi_spec \
+    ENTRYPOINT st run --include-path-regex '^/api/v1/health/' \
+                    --exclude-path '/api/v1/health/inspection' \ #excluding since this is a internal debug endpoint
+                    --checks=all $openapi_spec \
                     --workers=2 \
                     --wait-for-schema=120 \
                     --max-response-time=5000 \
                     --hypothesis-max-examples=1000 \
                     --data-generation-method=all \
-                    --skip-deprecated-operations \
+                    --exclude-deprecated \
                     --force-schema-version=30 \
                     --show-trace \
                     --force-color  \
@@ -31,7 +35,7 @@ package-schemathesis:
 # test-fuzzer-api - Fuzzy test cat-gateway using openapi specs.
 test-fuzzer-api:
     FROM earthly/dind:alpine-3.19-docker-25.0.5-r0
-    RUN apk update && apk add iptables-legacy # workaround for https://github.com/earthly/earthly/issues/3784
+    RUN apk update && apk add iptables-legacy curl # workaround for https://github.com/earthly/earthly/issues/3784
     COPY schemathesis-docker-compose.yml .
     LET OPENAPI_SPEC="http://0.0.0.0:3030/docs/cat-gateway.json"
     ARG seed
@@ -45,11 +49,15 @@ test-fuzzer-api:
         --service cat-gateway \
         --allow-privileged
 
-        RUN docker run --net=host --name=st schemathesis:latest && \
+        RUN docker run --net=host --name=st schemathesis:latest || echo fail > fail; \
             docker cp st:/results/junit-report.xml junit-report.xml && \
             docker cp st:/results/cassette.yaml cassette.yaml
     END
     WAIT
         SAVE ARTIFACT junit-report.xml AS LOCAL schemathesis.junit-report.xml
         SAVE ARTIFACT cassette.yaml AS LOCAL cassette.yaml
+    END
+    IF [ -f fail ]
+        RUN --no-cache echo "Schemathesis test failed. Check the logs for more details" && \
+            exit 1
     END