Define GitHub actions workflows for building and testing dbt

.github/actions/install_dbt_requirements/action.yaml

@@ -0,0 +1,27 @@
+name: Install dbt dependencies
+description: Installs Python and dbt requirements for a workflow
+inputs:
+  dbt_project_dir:
+    description: Path to the directory containing the dbt project.
+    required: false
+    default: ./dbt
+  requirements_file_path:
+    description: Path to Python requirements file.
+    required: false
+    default: ./dbt/requirements.txt
+runs:
+  using: composite
+  steps:
+      - name: Setup python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.x
+
+      - name: Install python requirements
+        run: python -m pip install -r ${{ inputs.requirements_file_path }}
+        shell: bash
+
+      - name: Install dbt requirements
+        run: dbt deps
+        working-directory: ${{ inputs.dbt_project_dir }}
+        shell: bash

.github/actions/load_environment_variables/action.yaml

@@ -0,0 +1,15 @@
+name: Load environment variables
+description: Configures environment variables for a workflow
+inputs:
+  env_var_file_path:
+    description: |
+      File path to variable file or directory.
+      Defaults to ./.github/variables/* if none specified
+      and runs against each file in that directory.
+    required: false
+    default: ./.github/variables/*
+runs:
+  using: composite
+  steps:
+    - run: sed "" ${{ inputs.env_var_file_path }} >> "$GITHUB_ENV"
+      shell: bash

.github/scripts/cleanup_dbt_resources.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# Clean up dbt resources created by a CI run or by local development.
+#
+# Takes one argument representing the target environment to clean up,
+# one of `dev` or `ci`. E.g.:
+#
+# ./cleanup_dbt_resources.sh dev
+#
+# Assumes that jq is installed and available on the caller's path.
+set -euo pipefail
+
+if [[ "$#" -eq 0 ]]; then
+    echo "Missing first argument representing dbt target"
+    exit 1
+fi
+
+if [ "$1" == "prod" ]; then
+    echo "Target cannot be 'prod'"
+    exit 1
+fi
+
+schemas_json=$(dbt --quiet list --resource-type model --target "$1" \
+    --output json --output-keys schema) || (echo "Error in dbt call" && exit 1)
+schemas=$(echo "$schemas_json"| sort | uniq | jq ' .schema') || (\
+    echo "Error in schema parsing" && exit 1
+)
+
+echo "Deleting the following schemas from Athena:"
+echo
+echo "$schemas"
+
+echo "$schemas" | xargs -i bash -c 'aws glue delete-database --name {} || exit 255'
+
+echo
+echo "Done!"

.github/variables/dbt.env

@@ -0,0 +1,3 @@
+CACHE_NAME=dbt-cache
+MANIFEST_DIR=dbt/target
+PROJECT_DIR=dbt

.github/workflows/build_and_test_dbt.yaml

@@ -0,0 +1,122 @@
+name: Build and test dbt
+
+on:
+  pull_request:
+    branches: [master, data-catalog]
+    # Specifying event types manually allows us to run this flow when the
+    # PR is closed so that we can clean up staging dbt resources
+    types:
+      - opened
+      - synchronize
+      - closed
+      - reopened
+  push:
+    branches: [master, data-catalog]
+
+jobs:
+  build-and-test-dbt:
+    runs-on: ubuntu-latest
+    # These permissions are needed to interact with GitHub's OIDC Token endpoint
+    # so that we can authenticate with AWS
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install dbt requirements
+        uses: ./.github/actions/install_dbt_requirements
+
+      - name: Load environment variables
+        uses: ./.github/actions/load_environment_variables
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: ${{ secrets.AWS_IAM_ROLE_TO_ASSUME_ARN }}
+          aws-region: us-east-1
+
+      - name: Set environment for branch
+        run: |
+          if [[ $GITHUB_REF_NAME == 'master' ]]; then
+            echo "On master branch"
+            {
+              echo "TARGET=prod";
+              echo "CACHE_KEY=master";
+            } >> "$GITHUB_ENV"
+          elif [[ $GITHUB_REF_NAME == 'data-catalog' ]]; then
+            echo "On data catalog branch"
+            {
+              echo "TARGET=ci";
+              echo "CACHE_KEY=data-catalog";
+              echo "GITHUB_HEAD_REF=data-catalog";
+            } >> "$GITHUB_ENV"
+          else
+            echo "On pull request branch"
+            {
+              echo "TARGET=ci";
+              echo "CACHE_KEY=$GITHUB_HEAD_REF";
+            } >> "$GITHUB_ENV"
+          fi
+        shell: bash
+
+      - name: Cache dbt manifest
+        id: cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.MANIFEST_DIR }}
+          key: ${{ env.CACHE_NAME }}-${{ env.CACHE_KEY }}
+          restore-keys: |
+            ${{ env.CACHE_NAME }}-data-catalog
+            ${{ env.CACHE_NAME }}-master
+
+      - if: ${{ steps.cache.outputs.cache-hit == 'true' }}
+        name: Set command args to build/test modified resources
+        run: echo "MODIFIED_RESOURCES_ONLY=true" >> "$GITHUB_ENV"
+        shell: bash
+
+      - if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+        name: Set command args to build/test all resources
+        run: echo "MODIFIED_RESOURCES_ONLY=false" >> "$GITHUB_ENV"
+        shell: bash
+
+      - name: Test dbt macros
+        run: dbt run-operation test_all
+        working-directory: ${{ env.PROJECT_DIR }}
+        shell: bash
+
+      - name: Build models
+        run: |
+          if [[ $MODIFIED_RESOURCES_ONLY == 'true' ]]; then
+            echo "Running build on modified resources only"
+            dbt run --target "$TARGET" -s state:modified --defer --state target/
+          else
+            echo "Running build on all resources"
+            dbt run --target "$TARGET"
+          fi
+        working-directory: ${{ env.PROJECT_DIR }}
+        shell: bash
+
+      - name: Test models
+        run: |
+          if [[ $MODIFIED_RESOURCES_ONLY == 'true' ]]; then
+            echo "Running tests on modified resources only"
+            dbt test --target "$TARGET" -s state:modified --state target/
+          else
+            echo "Running tests on all resources"
+            dbt test --target "$TARGET"
+          fi
+        working-directory: ${{ env.PROJECT_DIR }}
+        shell: bash
+
+      - if: ${{ github.event_name == 'pull_request' && github.event.action == 'closed' }}
+        name: Install requirements for cleaning up dbt resources
+        run: sudo apt-get update && sudo apt-get install jq
+        shell: bash
+
+      - if: ${{ github.event_name == 'pull_request' && github.event.action == 'closed' }}
+        name: Clean up dbt resources
+        run: ../.github/scripts/cleanup_dbt_resources.sh ci
+        working-directory: ${{ env.PROJECT_DIR }}
+        shell: bash

.github/workflows/test_dbt_models.yaml

@@ -0,0 +1,32 @@
+name: Test dbt models
+
+on: workflow_dispatch
+
+jobs:
+  test-dbt-models:
+    runs-on: ubuntu-latest
+    # These permissions are needed to interact with GitHub's OIDC Token endpoint
+    # so that we can authenticate with AWS
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install dbt requirements
+        uses: ./.github/actions/install_dbt_requirements
+
+      - name: Load environment variables
+        uses: ./.github/actions/load_environment_variables
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: ${{ secrets.AWS_IAM_ROLE_TO_ASSUME_ARN }}
+          aws-region: us-east-1
+
+      - name: Test models
+        run: dbt test --target ci
+        working-directory: ${{ env.PROJECT_DIR }}
+        shell: bash

dbt/models/default/schema.yml

@@ -31,15 +31,15 @@ models:
             - pin
             - year
           config:
-            error_if: ">280655"
+            error_if: ">280659"
       # Unique by case number and year
       - unique_combination_of_columns:
           name: vw_pin_appeal_unique_by_case_number_and_year
           combination_of_columns:
             - year
             - case_no
           config:
-            error_if: ">365779"
+            error_if: ">365855"
       # `change` should be an enum
       - dbt_utils.expression_is_true:
           name: vw_pin_appeal_no_unexpected_change_values
@@ -85,7 +85,7 @@ models:
               case when char_renovation = '1' then true else false end
             )
           config:
-            error_if: ">73925"
+            error_if: ">73941"
       # TODO: Characteristics columns should adhere to pre-determined criteria
   - name: vw_pin_address_test
     description: '{{ doc("vw_pin_address_test") }}'
@@ -111,7 +111,7 @@ models:
             - mail_address_zipcode_1
             - mail_address_zipcode_2
           config:
-            error_if: ">879261"
+            error_if: ">880581"
       # TODO: Mailing address changes after validated sale(?)
       # TODO: Site addresses are all in Cook County
   - name: vw_pin_condo_char_test