Integration tests for prod config

.github/get_actual_json_path.sh

@@ -17,6 +17,12 @@ sorted_version_numbers=($(printf '%s\n' "${version_numbers[@]}" | sort -rn))
 
 # Retrieve the latest version folder
 latest_version_number=${sorted_version_numbers[0]}
+
+# Check if second argument exists and if so, subtract 1 from the latest version
+if [ -n "$2" ]; then
+  latest_version_number=$((latest_version_number - 1))
+fi
+
 latest_version_folder="v${latest_version_number}"
 
 echo $latest_version_folder

.github/workflows/run_integration_tests.yaml

@@ -0,0 +1,135 @@
+name: Integration tests for config
+
+on:
+  pull_request:
+    paths:
+      - "chains/**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  BODY_FILE: comment-body.txt
+
+jobs:
+  prepare-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+      current: ${{ steps.current.outputs.any_changed }}
+      previous: ${{ steps.previous.outputs.any_changed }}
+    name: Check which version was changed
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: 🛠 Set up actual paths
+        uses: ./.github/workflows/setup-path
+
+      - name: Was chains.json changed?
+        id: current
+        uses: tj-actions/[email protected]
+        with:
+          files: |
+            ${{ env.CHAINS_JSON_PATH }}
+
+      - name: Was previous chains.json changed?
+        id: previous
+        uses: tj-actions/[email protected]
+        with:
+          files: |
+            ${{ env.PREVIOUS_CHAINS_JSON_PATH }}
+
+      - name: Generate test paths
+        run: |
+          if [[ "${{ steps.current.outputs.any_changed }}" == 'true' ]]; then
+            echo "TEST_PATHS=${{ env.CHAINS_JSON_PATH }}" >> $GITHUB_ENV
+          fi
+          if [[ "${{ steps.previous.outputs.any_changed }}" == 'true' ]]; then
+            if [[ -n "${{ env.TEST_PATHS }}" ]]; then
+              echo "TEST_PATHS=${{ env.TEST_PATHS }},${{ env.PREVIOUS_CHAINS_JSON_PATH }}" >> $GITHUB_ENV
+            else
+              echo "TEST_PATHS=${{ env.PREVIOUS_CHAINS_JSON_PATH }}" >> $GITHUB_ENV
+            fi
+          fi
+      - name: Set matrix
+        id: set-matrix
+        run: |
+          echo "::set-output name=matrix::{\"test_path\": [\"${TEST_PATHS}\"]}"
+
+
+  integration-tests:
+    name: ${{ matrix.test_path }}
+    runs-on: ubuntu-latest
+    needs: prepare-matrix
+    if: always() && (needs.prepare-matrix.outputs.current == 'true' || needs.prepare-matrix.outputs.previous == 'true')
+    strategy:
+      fail-fast: false
+      matrix:
+        test_path: ${{fromJson(needs.prepare-matrix.outputs.matrix).test_path}}
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up actual paths
+        uses: ./.github/workflows/setup-path
+
+      - name: Install dependencies
+        run: make init
+
+      - name: Run test
+        run: CHAINS_JSON_PATH=${{ matrix.test_path }} make test-core
+        continue-on-error: true
+
+      - name: Surface failing tests
+        if: always()
+        uses: pmeier/pytest-results-action@main
+        with:
+          path: test-results.xml
+
+  process-results:
+    runs-on: ubuntu-latest
+    needs: integration-tests
+    if: always()
+    steps:
+      - name: Save URLs to file
+        run: echo "🧪 Test results" > ${{ env.BODY_FILE }}
+
+      - name: Install jq
+        run: sudo apt-get install jq
+
+      - name: Get job IDs and create URLs
+        id: get-job-ids
+        run: |
+          url="https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs"
+          response=$(curl --silent --location "$url")
+          job_ids=$(echo "$response" | jq '.jobs[1:-1][] | .id')
+          urls=""
+          for id in $job_ids
+          do
+            echo "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/1#summary-$id" >> ${{ env.BODY_FILE }}
+          done
+
+      - name: Find Comment
+        uses: peter-evans/find-comment@v2
+        id: fc
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-author: 'github-actions[bot]'
+          body-includes: 🧪 Test results
+
+      - name: Create comment
+        if: steps.fc.outputs.comment-id == ''
+        uses: peter-evans/create-or-update-comment@v2
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          body-file: ${{ env.BODY_FILE }}
+
+      - name: Update comment
+        if: steps.fc.outputs.comment-id != ''
+        uses: peter-evans/create-or-update-comment@v2
+        with:
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          body-file: ${{ env.BODY_FILE }}
+          edit-mode: replace

.github/workflows/setup-path/action.yml

@@ -6,13 +6,16 @@ runs:
     - name: Setup version to variables
       run: |
         echo "CHAINS_VERSION=$(bash .github/get_actual_json_path.sh chains)" >> "$GITHUB_ENV"
+        echo "PREVIOUS_CHAINS_VERSION=$(bash .github/get_actual_json_path.sh chains 1)" >> "$GITHUB_ENV"
         echo "XCM_VERSION=$(bash .github/get_actual_json_path.sh xcm)" >> "$GITHUB_ENV"
       shell: bash
 
     - name: Setup json paths
       run: |
         echo "DEV_CHAINS_JSON_PATH=chains/${{ env.CHAINS_VERSION }}/chains_dev.json" >> "$GITHUB_ENV"
         echo "CHAINS_JSON_PATH=chains/${{ env.CHAINS_VERSION }}/chains.json" >> "$GITHUB_ENV"
+        echo "PREVIOUS_DEV_CHAINS_JSON_PATH=chains/${{ env.PREVIOUS_CHAINS_VERSION }}/chains_dev.json" >> "$GITHUB_ENV"
+        echo "PREVIOUS_CHAINS_JSON_PATH=chains/${{ env.PREVIOUS_CHAINS_VERSION }}/chains.json" >> "$GITHUB_ENV"
         echo "DEV_XCM_JSON_PATH=xcm/${{ env.XCM_VERSION }}/transfers_dev.json" >> "$GITHUB_ENV"
         echo "XCM_JSON_PATH=xcm/${{ env.XCM_VERSION }}/transfers.json" >> "$GITHUB_ENV"
       shell: bash

makefile

@@ -8,6 +8,7 @@ RE_RUNS = 2
 RE_RUN_DELAY = 15
 ALLURE_DIR = allure-results
 TEST_RUN = $(VENV)/bin/python -m pytest --rootdir . --alluredir=$(ALLURE_DIR) -n $(THREADS) -v --reruns $(RE_RUNS) --reruns-delay $(RE_RUN_DELAY)
+TEST_RUN_JUNIT = $(VENV)/bin/python -m pytest --rootdir . --junit-xml=test-results.xml -n $(THREADS) -v --reruns $(RE_RUNS) --reruns-delay $(RE_RUN_DELAY)
 
 CHAINS_FILES=\
 	chains
@@ -49,6 +50,9 @@ requirements:
 
 test-all: test-nodes-availability test-networks-precision test-network-chain-id test-network-prefix test-eth-availability test-new-assets
 
+test-core:
+	CHAINS_JSON_PATH=$(CHAINS_JSON_PATH) $(TEST_RUN_JUNIT) -m core
+
 test-nodes-availability:
 	$(TEST_RUN) "./tests/test_nodes_availability.py"