Add extract-chicago-permits workflow

.github/workflows/extract-chicago-permits.yaml

@@ -0,0 +1,98 @@
+# Workflow that can be manually dispatched to trigger Chicago permit extraction.
+
+name: extract-chicago-permits
+
+on:
+  workflow_dispatch:
+
+env:
+  WORKING_DIR: chicago
+  S3_BUCKET: ccao-data-public-us-east-1
+  S3_PREFIX: permits/chicago
+
+jobs:
+  extract-chicago-permits:
+    runs-on: ubuntu-latest
+    # These permissions are needed to interact with GitHub's OIDC Token endpoint
+    # so that we can authenticate with AWS
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Checkout repo code
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.10
+          cache: pipenv
+
+      - name: Install pipenv
+        run: pip install pipenv
+        shell: bash
+
+      - name: Install Python requirements
+        run: pipenv install
+        shell: bash
+        working-directory: ${{ env.WORKING_DIR }}
+
+      - name: Extract permits
+        run: python3 permit_cleaning.py
+        shell: bash
+        working-directory: ${{ env.WORKING_DIR }}
+
+      - name: Compress permit directories into one file
+        id: compress-permits
+        run: |
+          ZIP_FILENAME="chicago-permits-$(date +%Y%m%d%H%M%S).zip"
+          mkdir chicago-permits
+          mv csvs_for_* chicago-permits/
+          zip -r "$ZIP_FILENAME" chicago-permits
+          echo "filename=$ZIP_FILENAME" >> "$GITHUB_OUTPUT"
+        shell: bash
+        working-directory: ${{ env.WORKING_DIR }}
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.AWS_IAM_ROLE_TO_ASSUME_ARN }}
+          aws-region: us-east-1
+
+      - name: Upload compressed permit file to S3
+        id: s3-upload
+        run: |
+          S3_PATH="s3://${S3_BUCKET}/${S3_PREFIX}/${PERMIT_FILENAME}"
+          aws s3 cp "$PERMIT_FILENAME" "$S3_PATH"
+          S3_URL="https://${S3_BUCKET}.s3.amazonaws.com/${S3_PREFIX}/${PERMIT_FILENAME}"
+          echo "Permit file successfully uploaded to S3: $S3_URL"
+          echo "url=$S3_URL" >> "$GITHUB_OUTPUT"
+        shell: bash
+        working-directory: ${{ env.WORKING_DIR }}
+        env:
+          PERMIT_FILENAME: ${{ steps.compress-permits.outputs.filename }}
+
+      - name: Mask SNS topic ARN
+        # This step is necessary so that AWS doesn't accidentally log the ARN
+        # in case the `aws sns publish` command fails
+        run: echo "::add-mask::${{ secrets.AWS_SNS_TOPIC_ARN }}"
+        shell: bash
+
+      - name: Publish to SNS notification topic
+        run: |
+          SUBJECT="New Chicago permit data for $(date +%d/%m/%Y)"
+          MESSAGE=$(cat <<EOF
+          New Chicago permit data has been extracted on $(date +%d/%m/%Y).
+
+          Download a .zip file containing the permit data here:
+          $S3_URL
+          EOF
+          )
+
+          aws sns publish \
+            --topic-arn "$SNS_TOPIC_ARN" \
+            --subject "$SUBJECT" \
+            --message "$MESSAGE"
+        env:
+          S3_URL: ${{ steps.s3-upload.outputs.url }}
+          SNS_TOPIC_ARN: ${{ secrets.AWS_SNS_TOPIC_ARN }}

.gitignore

@@ -158,3 +158,6 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+# Ignore generated data
+chicago/csvs_for_*

chicago/Pipfile

@@ -0,0 +1,14 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+requests = "2.31.*"
+pandas = "2.1.*"
+sodapy = "2.2.*"
+
+[dev-packages]
+
+[requires]
+python_version = "3.10"