test(cat-gateway): adds nightly schemathesis (#1504)

* wip * wip * wip * wip * wip * wip * fix: remove scylla earthfile * feat: runs schemathesis only on api/started endpoint * clean up * fix: blueprint * fix: blueprint * fix: blueprint * fix: blueprint * fix: remove hooks * fix: remove some checks * feat: adds seed param to chemathesis package * fix: poetry * fix: earthly dockerd bug * fix: remove checks all param * fix: remove duplicated function * fix: regex * adds all check * chore: clean up * clean up * chore: update schemathesis version * fix: update schemathesis option to latest version * fixes * fix: st target failing statment * feat: adds schemathesis test target * test run * test run * remove test values --------- Co-authored-by: Steven Johnson <[email protected]> Co-authored-by: Oleksandr Prokhorenko <[email protected]>
input-output-hk · Jan 16, 2025 · 56454b6 · 56454b6
1 parent 21263b1
commit 56454b6
Show file tree

Hide file tree

Showing 2 changed files with 68 additions and 7 deletions.
diff --git a/catalyst-gateway/tests/schemathesis_tests/Earthfile b/catalyst-gateway/tests/schemathesis_tests/Earthfile
@@ -3,19 +3,19 @@ VERSION 0.8
 package-schemathesis:
     FROM python:3.12-alpine3.20
     # TODO: https://github.com/input-output-hk/catalyst-voices/issues/465
-    ARG openapi_spec
-    # optional argument that can be used to pass a --hypothesis-seed to replicate specific test failures
-    ARG seed
+    ARG api_spec
+    ARG seed # optional argument that can be used to pass a --hypothesis-seed to replicate specific test failures
     ARG version=3.39.5
 
     RUN apk add --no-cache gcc musl-dev
     RUN python -m pip install schemathesis==$version
     RUN mkdir /results
     COPY ./hooks/hooks.py .
     VOLUME /results
-    ENTRYPOINT st run --include-path-regex '^/api/v1/health/' \
+
+    ENTRYPOINT st run --exclude-path-regex 'draft' \
                     --exclude-path '/api/v1/health/inspection' \ #excluding since this is a internal debug endpoint
-                    $openapi_spec \
+                    $api_spec \
                     --workers=2 \
                     --wait-for-schema=500 \
                     --max-response-time=5000 \
@@ -37,12 +37,12 @@ test-fuzzer-api:
     FROM earthly/dind:alpine-3.19-docker-25.0.5-r0
     RUN apk update && apk add iptables-legacy curl # workaround for https://github.com/earthly/earthly/issues/3784
     COPY schemathesis-docker-compose.yml .
-    LET OPENAPI_SPEC="http://0.0.0.0:3030/docs/cat-gateway.json"
+    LET api_spec="http://0.0.0.0:3030/docs/cat-gateway.json"
     ARG seed
 
     WITH DOCKER \
         --compose schemathesis-docker-compose.yml \
-        --load schemathesis:latest=(+package-schemathesis --openapi_spec=$OPENAPI_SPEC --seed=$seed) \
+        --load schemathesis:latest=(+package-schemathesis --api_spec=$api_spec --seed=$seed) \
         --load event-db:latest=(../../event-db+build) \
         --load cat-gateway:latest=(../+package-cat-gateway-integration) \
         --service event-db \
@@ -60,4 +60,64 @@ test-fuzzer-api:
     IF [ -f fail ]
         RUN --no-cache echo "Schemathesis test failed. Check the logs for more details" && \
             exit 1
+    END
+
+nightly-package-schemathesis:
+    FROM python:3.12-alpine3.20
+    # TODO: https://github.com/input-output-hk/catalyst-voices/issues/465
+    ARG api_spec
+    # optional argument that can be used to pass a --hypothesis-seed to replicate specific test failures
+    ARG seed
+    ARG version=3.39.5
+
+    RUN apk add --no-cache gcc musl-dev
+    RUN python -m pip install schemathesis==$version
+    RUN mkdir /results
+    COPY ./hooks/hooks.py .
+    VOLUME /results
+    ENTRYPOINT st run --checks=all $api_spec \
+                    --workers=2 \
+                    --wait-for-schema=120 \
+                    --max-response-time=300 \
+                    --hypothesis-max-examples=1000 \
+                    --data-generation-method=all \
+                    --exclude-deprecated \
+                    --force-schema-version=30 \
+                    --show-trace \
+                    --force-color  \
+                    --junit-xml=/results/junit-report.xml \
+                    --cassette-path=/results/cassette.yaml \
+                    $seed
+
+    ARG tag="latest"
+    SAVE IMAGE schemathesis:$tag
+
+# nightly-test-fuzzer-api - Fuzzy test cat-gateway using openapi specs.
+nightly-test-fuzzer-api:
+    FROM earthly/dind:alpine-3.19-docker-25.0.5-r0
+    RUN apk update && apk add iptables-legacy curl # workaround for https://github.com/earthly/earthly/issues/3784
+    COPY schemathesis-docker-compose.yml .
+    LET api_spec="http://0.0.0.0:3030/docs/cat-gateway.json"
+    ARG seed
+
+    WITH DOCKER \
+        --compose schemathesis-docker-compose.yml \
+        --load schemathesis:latest=(+nightly-package-schemathesis --api_spec=$api_spec --seed=$seed) \
+        --load event-db:latest=(../../event-db+build) \
+        --load cat-gateway:latest=(../+package-cat-gateway-integration) \
+        --service event-db \
+        --service cat-gateway \
+        --allow-privileged
+
+        RUN --no-cache docker run --net=host --name=st schemathesis:latest || echo fail > fail; \
+            docker cp st:/results/junit-report.xml junit-report.xml && \
+            docker cp st:/results/cassette.yaml cassette.yaml
+    END
+    WAIT
+        SAVE ARTIFACT junit-report.xml AS LOCAL schemathesis-nightly.junit-report.xml
+        SAVE ARTIFACT cassette.yaml AS LOCAL cassette.yaml
+    END
+    IF [ -f fail ]
+        RUN --no-cache echo "Nightly schemathesis test failed. Check the logs for more details" && \
+            exit 1
     END
diff --git a/catalyst-gateway/tests/schemathesis_tests/blueprint.cue b/catalyst-gateway/tests/schemathesis_tests/blueprint.cue
@@ -4,6 +4,7 @@ project: {
 	ci: {
     targets: {
       "test-fuzzer-api": privileged: true
+      "nightly-test-fuzzer-api": privileged: true
     }
   }
 }