Performance Test Set-up (1) (#197)

.github/workflows/e2e.yml

@@ -5,6 +5,10 @@ on:
     branches: [master]
   pull_request:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   test:
     name: Test

.github/workflows/performance.yaml

@@ -0,0 +1,141 @@
+name: Performance Test K8s-snap
+
+on:
+  push:
+    branches: ["master"]
+  pull_request:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  BASE_SHA: ${{ github.before || github.event.pull_request.base.sha }}
+  BASE_BRANCH: ${{ github.base_ref || github.ref }}
+  TARGET_SHA: ${{ github.sha }}
+
+jobs:
+  build:
+    name: K8s-snap Performance Test
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@v2
+        with:
+          egress-policy: audit
+      - name: Checking out repo
+        uses: actions/checkout@v4
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - name: Install tox
+        run: |
+          pip install tox
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.22"
+      - name: Install lxd
+        run: |
+          sudo snap refresh lxd --channel 5.21/stable
+          sudo lxd init --auto
+          sudo usermod --append --groups lxd $USER
+          sg lxd -c 'lxc version'
+      - name: Ensure lxd network traffic flows by removing docker if installed
+        run: |
+          if command -v docker >/dev/null 2>&1; then
+            echo "Docker is installed, purging it"
+            sudo apt-get purge -y docker-engine docker docker.io docker-ce docker-ce-cli containerd runc
+          fi
+      - name: Download latest k8s-snap
+        run: |
+          sudo snap download k8s --channel=latest/edge --basename k8s
+      - name: Unpack Snap
+        run: |
+          sudo unsquashfs -d snap-unpack-dir k8s.snap
+      - name: Create snap with k8s-dqlite ${{ github.head_ref }}
+        run: |
+          make static
+          sudo cp ./bin/static/k8s-dqlite snap-unpack-dir/bin/k8s-dqlite
+          sudo chmod o+r snap-unpack-dir/bin/k8s-dqlite
+          sudo mksquashfs snap-unpack-dir head.snap -noappend -comp lzo -no-fragments
+      - name: Run Performance test ${{ github.head_ref }} snap
+        env:
+          TEST_SNAP: ${{ github.workspace }}/head.snap
+          TEST_SUBSTRATE: lxd
+          TEST_LXD_IMAGE: ubuntu:22.04
+          TEST_INSPECTION_REPORTS_DIR: ${{ github.workspace }}/inspection-reports
+        run: |
+          cd test/performance && sg lxd -c 'tox -e performance'
+      - name: Create snap with k8s-dqlite base code
+        run: |
+          set -o pipefail
+          git fetch origin $BASE_BRANCH
+          git reset --hard $BASE_SHA
+          make static
+          sudo cp ./bin/static/k8s-dqlite snap-unpack-dir/bin/k8s-dqlite
+          sudo chmod o+r snap-unpack-dir/bin/k8s-dqlite
+          sudo mksquashfs snap-unpack-dir base-code.snap -noappend -comp lzo -no-fragments
+      - name: Switch back to target branch
+        run: git reset --hard $TARGET_SHA
+      - name: Run Performance test for base code snap
+        env:
+          TEST_SNAP: ${{ github.workspace }}/base-code.snap
+          TEST_SUBSTRATE: lxd
+          TEST_LXD_IMAGE: ubuntu:22.04
+          TEST_INSPECTION_REPORTS_DIR: ${{ github.workspace }}/inspection-reports
+        run: |
+          cd test/performance && sg lxd -c 'tox -e performance'
+      - name: Create snap with k8s-dqlite v1.1.11
+        run: |
+          set -o pipefail
+          git fetch origin --tags
+          git reset --hard v1.1.11
+          make static
+          sudo cp ./bin/static/k8s-dqlite snap-unpack-dir/bin/k8s-dqlite
+          sudo chmod o+r snap-unpack-dir/bin/k8s-dqlite
+          sudo mksquashfs snap-unpack-dir v1-1-11.snap -noappend -comp lzo -no-fragments
+      - name: Switch back to target branch
+        run: git reset --hard $TARGET_SHA
+      - name: Run Performance test for v1.1.11 snap
+        env:
+          TEST_SNAP: ${{ github.workspace }}/v1-1-11.snap
+          TEST_SUBSTRATE: lxd
+          TEST_LXD_IMAGE: ubuntu:22.04
+          TEST_INSPECTION_REPORTS_DIR: ${{ github.workspace }}/inspection-reports
+        run: |
+          cd test/performance && sg lxd -c 'tox -e performance'
+      - name: Create snap with k8s-dqlite v1.2.0
+        run: |
+          set -o pipefail
+          git fetch origin --tags
+          git reset --hard v1.2.0
+          make static
+          sudo cp ./bin/static/k8s-dqlite snap-unpack-dir/bin/k8s-dqlite
+          sudo chmod o+r snap-unpack-dir/bin/k8s-dqlite
+          sudo mksquashfs snap-unpack-dir v1-2-0.snap -noappend -comp lzo -no-fragments
+      - name: Switch back to target branch
+        run: git reset --hard $TARGET_SHA
+      - name: Run Performance test for v1.2.0 snap
+        env:
+          TEST_SNAP: ${{ github.workspace }}/v1-2-0.snap
+          TEST_SUBSTRATE: lxd
+          TEST_LXD_IMAGE: ubuntu:22.04
+          TEST_INSPECTION_REPORTS_DIR: ${{ github.workspace }}/inspection-reports
+        run: |
+          cd test/performance && sg lxd -c 'tox -e performance'
+      - name: Prepare inspection reports
+        if: failure()
+        run: |
+          tar -czvf inspection-reports.tar.gz -C ${{ github.workspace }} inspection-reports
+          echo "artifact_name=inspection-reports-${{ matrix.os }}" | sed 's/:/-/g' >> $GITHUB_ENV
+      - name: Upload inspection report artifact
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.artifact_name }}
+          path: ${{ github.workspace }}/inspection-reports.tar.gz

test/performance/.copyright.tmpl

@@ -0,0 +1 @@
+Copyright ${years} ${owner}.

test/performance/Readme.md

@@ -0,0 +1,60 @@
+# Performance Testing
+
+## Overview
+
+End to end tests are written in Python. They are built on top of a [Harness](./tests/conftest.py) fixture so that they can run on multiple environments like LXD or in the future on the local machine.
+
+End to end tests can be configured using environment variables. You can see all available options in [./tests/config.py](./tests/config.py).
+
+## Running end to end tests
+
+Running the end to end tests requires `python3` and `tox`. Install with:
+
+```bash
+sudo apt install python3-virtualenv
+virtualenv .venv
+. .venv/bin/activate
+pip install 'tox<5'
+```
+
+Further, make sure that you have downloaded the `k8s.snap`:
+
+```bash
+sudo snap download k8s --channel=latest/edge --basename k8s
+```
+
+In general, all end to end tests will require specifying the local path to the snap package under test, using the `TEST_SNAP` environment variable. Make sure to specify the full path to the file.
+
+End to end tests are typically run with: `cd test/performance && tox -e performance`
+
+### Running end to end tests on the local machine
+
+```bash
+export TEST_SNAP=$PWD/k8s.snap
+export TEST_SUBSTRATE=local
+
+cd test/performance && tox -e performance
+```
+
+> *NOTE*: When running locally, end to end tests that create more than one instance will fail.
+
+### Running end to end tests on LXD containers
+
+First, make sure that you have initialized LXD:
+
+```bash
+sudo lxd init --auto
+```
+
+Then, run the tests with:
+
+```bash
+export TEST_SNAP=$PWD/k8s.snap
+export TEST_SUBSTRATE=lxd
+
+export TEST_LXD_IMAGE=ubuntu:22.04               # (optionally) specify which image to use for LXD containers
+export TEST_LXD_PROFILE_NAME=k8s-performance     # (optionally) specify profile name to configure
+export TEST_SKIP_CLEANUP=1                       # (optionally) do not destroy machines after tests finish
+
+cd test/performance && tox -e performance
+```

test/performance/lxd-profile.yaml

@@ -0,0 +1,105 @@
+description: "LXD profile for Canonical Kubernetes"
+config:
+  linux.kernel_modules: ip_vs,ip_vs_rr,ip_vs_wrr,ip_vs_sh,ip_tables,ip6_tables,iptable_raw,netlink_diag,nf_nat,overlay,br_netfilter,xt_socket
+  raw.lxc: |
+    lxc.apparmor.profile=unconfined
+    lxc.mount.auto=proc:rw sys:rw cgroup:rw
+    lxc.cgroup.devices.allow=a
+    lxc.cap.drop=
+  security.nesting: "true"
+  security.privileged: "true"
+devices:
+  aadisable2:
+    path: /dev/kmsg
+    source: /dev/kmsg
+    type: unix-char
+  dev-loop-control:
+    major: "10"
+    minor: "237"
+    path: /dev/loop-control
+    type: unix-char
+  dev-loop0:
+    major: "7"
+    minor: "0"
+    path: /dev/loop0
+    type: unix-block
+  dev-loop1:
+    major: "7"
+    minor: "1"
+    path: /dev/loop1
+    type: unix-block
+  dev-loop2:
+    major: "7"
+    minor: "2"
+    path: /dev/loop2
+    type: unix-block
+  dev-loop3:
+    major: "7"
+    minor: "3"
+    path: /dev/loop3
+    type: unix-block
+  dev-loop4:
+    major: "7"
+    minor: "4"
+    path: /dev/loop4
+    type: unix-block
+  dev-loop5:
+    major: "7"
+    minor: "5"
+    path: /dev/loop5
+    type: unix-block
+  dev-loop6:
+    major: "7"
+    minor: "6"
+    path: /dev/loop6
+    type: unix-block
+  dev-loop7:
+    major: "7"
+    minor: "7"
+    path: /dev/loop7
+    type: unix-block
+  dev-loop8:
+    major: "7"
+    minor: "8"
+    path: /dev/loop8
+    type: unix-block
+  dev-loop9:
+    major: "7"
+    minor: "9"
+    path: /dev/loop9
+    type: unix-block
+  dev-loop10:
+    major: "7"
+    minor: "10"
+    path: /dev/loop10
+    type: unix-block
+  dev-loop11:
+    major: "7"
+    minor: "11"
+    path: /dev/loop11
+    type: unix-block
+  dev-loop12:
+    major: "7"
+    minor: "12"
+    path: /dev/loop12
+    type: unix-block
+  dev-loop13:
+    major: "7"
+    minor: "13"
+    path: /dev/loop13
+    type: unix-block
+  dev-loop14:
+    major: "7"
+    minor: "14"
+    path: /dev/loop14
+    type: unix-block
+  dev-loop15:
+    major: "7"
+    minor: "15"
+    path: /dev/loop15
+    type: unix-block
+  dev-loop16:
+    major: "7"
+    minor: "16"
+    path: /dev/loop16
+    type: unix-block

test/performance/requirements-dev.txt

@@ -0,0 +1,5 @@
+black==24.3.0
+codespell==2.2.4
+flake8==6.0.0
+isort==5.12.0
+licenseheaders==0.8.8

test/performance/requirements-test.txt

@@ -0,0 +1,5 @@
+coverage[toml]==7.2.5
+pytest==7.3.1
+PyYAML==6.0.1
+tenacity==8.2.3
+pylint==3.2.5