diff --git a/.github/dependabot.yml b/.github/dependabot.yml
new file mode 100644
index 0000000..c963e3c
--- /dev/null
+++ b/.github/dependabot.yml
@@ -0,0 +1,12 @@
+version: 2
+updates:
+
+  - package-ecosystem: "github-actions"
+    commit-message:
+      include: "scope"
+      prefix: "Actions"
+    directory: "/"
+    labels:
+      - "enhancement"
+    schedule:
+      interval: "weekly"
diff --git a/.github/workflows/expected-output.txt b/.github/workflows/expected-output.txt
new file mode 100644
index 0000000..f285bf8
--- /dev/null
+++ b/.github/workflows/expected-output.txt
@@ -0,0 +1,32 @@
+--- /tmp/tmpcdry5mph	2024-04-21 19:36:45.813805199 +0200
++++ Caddyfile	2024-04-21 19:36:45.813805199 +0200
+@@ -0,0 +1,27 @@
++# NOTE: This file has been generated, do not edit
++(common) {
++    encode zstd gzip
++    log {
++        output stdout
++    }
++}
++
++example.org {
++    import common
++    reverse_proxy example-org:80
++}
++
++example.net {
++    import common
++    redir https://example.org{uri}
++}
++
++www.example.net {
++    import common
++    redir https://example.org{uri}
++}
++
++www.example.org {
++    import common
++    redir https://example.org{uri}
++}
+
+changed=yes comment='Caddyfile changed'
diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
new file mode 100644
index 0000000..d34d8dc
--- /dev/null
+++ b/.github/workflows/run-tests.yml
@@ -0,0 +1,36 @@
+# Copyright (c) 2024 Sebastian Pipping <sebastian@pipping.org>
+# Licensed under GNU Affero GPL v3 or later
+
+name: Run smoke test
+
+# Drop permissions to minimum for security
+permissions:
+  contents: read
+
+on:
+  pull_request:
+  push:
+  schedule:
+    - cron: '0 16 * * 5'  # Every Friday 4pm
+  workflow_dispatch:
+
+jobs:
+  smoke_test:
+    name: Run the test suite
+    strategy:
+      matrix:
+        python-version: [3.8, 3.12]  # no particular need for in-between versions
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633  # v4.1.2
+    - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d  # v5.1.0
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Run smoke test
+      run: |
+        set -x
+        python3 --version
+        ./Caddyfile.generate --config sites.cfg.EXAMPLE --saltstack > actual-output.txt
+        diff -u0 \
+            <(tail -n +3 .github/workflows/expected-output.txt) \
+            <(tail -n +3 actual-output.txt)