Merge pull request #2282 from DemocracyClub/uv2

Migrate to UV

.circleci/config.yml

@@ -23,7 +23,7 @@ jobs:
           node-version: '18'
 
       - restore_cache:
-          key: v3-dependencies-{{ checksum "requirements/base.txt" }}-{{ checksum "requirements/testing.txt" }}-{{ checksum "requirements/cdk.txt" }}
+          key: v3-dependencies-{{ checksum "uv.lock" }}
 
       - run:
           name: Install node node_modules
@@ -35,22 +35,15 @@ jobs:
           name: Install app dependencies
           command: |
             sudo apt update && sudo apt install -y gdal-bin python3-gdal python3-dev
-            pyenv local $(ls ~/.pyenv/versions/)
-            python -m venv .venv
+            deployscripts/install_uv.sh
+            uv venv --verbose --no-python-downloads
             . .venv/bin/activate
-            pip install --upgrade pip
-            pip install coveralls wheel
-            pip install -r requirements/testing.txt
+            uv sync --group cdk --group production
       - run:
           name: Install Playwright
           command: |
             . .venv/bin/activate
             playwright install
-      - run:
-          name: Install CDK Python dependencies
-          command: |
-            . .venv/bin/activate
-            pip install -r requirements/cdk.txt
       - run:
           name: Pip safety check
           command: |
@@ -60,7 +53,8 @@ jobs:
           paths:
             - ./.venv
             - ./node_modules
-          key: v3-dependencies-{{ checksum "requirements/base.txt" }}-{{ checksum "requirements/testing.txt" }}-{{ checksum "requirements/cdk.txt" }}
+            - $HOME/.cache/uv
+          key: v3-dependencies-{{ checksum "uv.lock" }}
 
       - run:
           name: Pre-test checks
@@ -70,7 +64,8 @@ jobs:
             python manage.py --version
             python manage.py check
             python manage.py makemigrations --check
-            pip check
+            uv lock --locked
+            uv pip check
             ruff format . --check
             ruff check .
             git ls-files '*.html' | xargs djhtml --check
@@ -85,7 +80,7 @@ jobs:
           name: Submit coverage
           command: |
             . .venv/bin/activate
-            coveralls
+            uv run --with coveralls coveralls
 
       - store_artifacts:
           path: test-results
@@ -107,20 +102,21 @@ jobs:
     steps:
     - checkout
     - restore_cache:
-        key: v3-machine-dependencies-{{ checksum "requirements/base.txt" }}-{{ checksum "requirements/testing.txt" }}-{{ checksum "requirements/cdk.txt" }}
+        key: v3-machine-dependencies-{{ checksum "uv.lock" }}
     - run:
         name: Install CDK Python dependencies
         command: |
-          pyenv local $(ls -1 /opt/circleci/.pyenv/versions | grep 3.12)
-          python -m venv .venv
+          deployscripts/install_uv.sh
+          uv venv --verbose --no-python-downloads
           . .venv/bin/activate
-          pip install -r requirements/cdk.txt
+          uv sync --only-group cdk
 
     - save_cache:
         paths:
           - ./.venv
           - ./node_modules
-        key: v3-machine-dependencies-{{ checksum "requirements/base.txt" }}-{{ checksum "requirements/testing.txt" }}-{{ checksum "requirements/cdk.txt" }}
+          - $HOME/.cache/uv
+        key: v3-machine-dependencies-{{ checksum "uv.lock" }}
 
     - run:
         name: CDK version
@@ -156,17 +152,16 @@ jobs:
     steps:
     - checkout
     - restore_cache:
-        key: v3-machine-dependencies-{{ checksum "requirements/base.txt" }}-{{ checksum "requirements/testing.txt" }}-{{ checksum "requirements/cdk.txt" }}
+        key: v3-machine-dependencies-{{ checksum "uv.lock" }}
     - run:
         name: CDK deploy
         command: |
-          pyenv local $(ls -1 /opt/circleci/.pyenv/versions | grep 3.12)
           . .venv/bin/activate
           npx cdk deploy --all --require-approval never --concurrency 3
     - save_cache:
         paths:
         - .cdk.out
-        key: v3-machine-dependencies-{{ checksum "requirements/base.txt" }}-{{ checksum "requirements/testing.txt" }}-{{ checksum "requirements/cdk.txt" }}
+        key: v3-machine-dependencies-{{ checksum "uv.lock" }}
     - slack/notify:
           event: fail
           template: basic_fail_1
@@ -192,11 +187,10 @@ jobs:
     - node/install:
         node-version: '18'
     - restore_cache:
-        key: v3-dependencies-{{ checksum "requirements/base.txt" }}-{{ checksum "requirements/testing.txt" }}-{{ checksum "requirements/cdk.txt" }}
+        key: v3-dependencies-{{ checksum "uv.lock" }}
     - run:
         name: "Code Deploy: Create deployment group"
         command: |
-          pyenv local $(ls ~/.pyenv/versions/)
           . .venv/bin/activate
           python deployscripts/create_deployment_group.py
     - run:

.github/dependabot.yml

@@ -3,14 +3,20 @@ updates:
 - package-ecosystem: pip
   directory: "/"
   schedule:
-    interval: daily
-  open-pull-requests-limit: 10
+    interval: "daily"
+  groups:
+    cdk-libs:
+      patterns:
+        - "aws-cdk*"
+        - "constructs"
+    boto-libs:
+      patterns:
+        - "boto*"
   ignore:
-  - dependency-name: boto3
+  - dependency-name: "boto*"
     update-types: ["version-update:semver-patch"]
-  - dependency-name: botocore
-    update-types: ["version-update:semver-patch"]
-  - dependency-name: django
+  - dependency-name: "django"
     versions:
-    - ">= 3.a"
-    - "< 4.2"
+      # We don't want notifications about Django 5
+      # until 5.2 LTS is released in 2025
+      - ">= 5.0"

.github/workflows/uv-lock.yml

@@ -0,0 +1,29 @@
+name: UV Lock
+
+on:
+  push:
+    branches:
+      - 'dependabot/**'
+
+permissions:
+  contents: write
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+
+    - run: ./deployscripts/install_uv.sh
+
+    - run: uv lock
+
+    - uses: stefanzweifel/git-auto-commit-action@v5
+      with:
+        commit_message: Update uv.lock

.safety-policy.yml

@@ -13,13 +13,7 @@ security: # configuration for the `safety check` command
         65213: # vulnerability in the POLY1305 MAC algorithm on PowerPC CPUs
             reason: we do not use the vulnerable function
             expires: '2025-01-20' # Bump if still applies
-        66742: # Don't do stupid things with Black. We don't. Remove when we go to ruff
-            reason: we do not use loads of leading tabs
-            expires: '2025-01-20' # Bump if still applies
-        67599: # Apparently all versions of pip are bad.
-            reason: We need to be able to pip install things.
-            expires: '2024-08-20' # Bump if still applies
         70612: #  jinja2 version 3.1.4
             reason: Only used in tests, no fixed version available, maintainer disputes it is a vuln
-            expires: '2024-10-20' # Bump if still applies
+            expires: '2025-01-20' # Bump if still applies
     continue-on-vulnerability-error: False # Suppress non-zero exit codes when vulnerabilities are found. Enable this in pipelines and CI/CD processes if you want to pass builds that have vulnerabilities

INSTALL.md

@@ -3,11 +3,13 @@ EveryElection requires Python 3.x and Postgres.
 
 ## Install Python dependencies
 
+Every Election uses [uv](https://docs.astral.sh/uv/) to manage python packages.
+[Install uv](https://docs.astral.sh/uv/getting-started/installation/) first if you don't already have it. Then
+
 ```
-pip install -U pip
-pip install -r requirements.txt
-pip install -r requirements/local.txt
+uv sync
 ```
+
 Gotcha: If you're having trouble installing psycopg2-binary on Apple silicon, set your library path before retrying the install:
 ```commandline
 export LIBRARY_PATH=$LIBRARY_PATH:/opt/homebrew/opt/openssl/lib

appspec.yml

@@ -73,6 +73,9 @@ hooks:
 # During the AfterInstall deployment lifecycle event, run the commands
 #   in the script specified in "location".
   AfterInstall:
+    - location: deployscripts/install_uv.sh
+      timeout: 300
+      runas: root
     - location: deployscripts/install_python_deps.sh
       timeout: 300
       runas: every_election

cdk_settings.json

@@ -1,4 +1,4 @@
 {
   "base_ami_id": "ami-0c40c7ef04b5ab961",
-  "recipe_version": "0.0.35"
+  "recipe_version": "0.0.41"
 }

cdk_stacks/components/add_user.yml

@@ -1,7 +1,7 @@
 name: AddUser
 description: Adds a system user
 schemaVersion: 1.0
-component_version: 0.0.4
+component_version: 0.0.5
 parameters:
   - username:
       type: string

cdk_stacks/components/install_app.yml

@@ -1,7 +1,7 @@
 name: InstallApp
 description: Installs the EE Django app
 schemaVersion: 1.0
-component_version: 0.0.22
+component_version: 0.0.27
 parameters:
   - git_branch:
       type: string
@@ -43,12 +43,11 @@ phases:
             - git checkout {{ git_branch }}
             - git pull --rebase origin {{ git_branch }}
             - git branch
-            - python3 -m venv .venv
+            - ./deployscripts/install_uv.sh
+            - uv venv
             - sudo chown -R {{ username }} /var/www/{{ username }}
             - . .venv/bin/activate
-            - pip install -U pip
-            - pip install -r requirements/production.txt
-            - pip install gevent
+            - uv sync --group production --no-group dev --no-group cdk
             - npm ci
       - name: CreateEnvWriterFile
         action: CreateFile

cdk_stacks/components/instance_connect.yml

@@ -1,7 +1,7 @@
 name: InstallInstanceConnect
 description: Installs AWS EC2 instance Connect
 schemaVersion: 1.0
-component_version: 0.0.2
+component_version: 0.0.3
 phases:
   - name: build
     steps:

deployscripts/install_python_deps.sh

@@ -3,8 +3,6 @@ set -xeE
 
 # should we delete the env and recreate?
 cd /var/www/every_election/repo/
-python3 -m venv .venv
+uv venv
 . .venv/bin/activate
-pip install -U pip
-pip install -r requirements/production.txt
-pip install gevent
+uv sync --group production --no-group dev --no-group cdk

deployscripts/install_uv.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -xeE
+
+UV_CONSTRAINT=">=0.4.27,<0.5.0"
+
+if [ "$CI" = "true" ]; then
+    pip install uv"$UV_CONSTRAINT"
+else
+    sudo PIP_BREAK_SYSTEM_PACKAGES=1 pip install uv"$UV_CONSTRAINT"
+fi

every_election/apps/elections/tests/test_admin.py

@@ -1,6 +1,6 @@
+from unittest import mock
 from unittest.mock import MagicMock
 
-import mock
 from django.test import TestCase
 from elections import admin
 from elections.models import ModerationHistory

every_election/apps/elections/tests/test_attach_posts_command.py

@@ -1,6 +1,6 @@
 from io import StringIO
+from unittest import mock
 
-import mock
 from django.test import TestCase
 from elections.management.commands import attach_posts_per_election_from_csv
 from elections.models import Election

every_election/apps/organisations/boundaries/boundary_bot/tests/test_scraper.py

@@ -1,6 +1,7 @@
+from unittest import mock
+
 from data_provider import base_data
 from django.test import TestCase
-from mock import mock
 from organisations.boundaries.boundary_bot.scraper import (
     LgbceScraper,
     ScraperException,

every_election/apps/organisations/boundaries/tests/test_osni_layer.py

@@ -1,4 +1,5 @@
-import mock
+from unittest import mock
+
 from django.contrib.gis.geos import MultiPolygon
 from django.test import TestCase
 from organisations.boundaries.osni import OsniLayer

every_election/apps/organisations/tests/test_admin.py

@@ -1,13 +1,13 @@
 import os
 from datetime import date
+from unittest.mock import Mock, patch
 
 import boto3
 from botocore.exceptions import ClientError
 from django.contrib.auth import get_user_model
 from django.test import TestCase, override_settings
 from django.urls import reverse
 from elections.tests.factories import ElectedRoleFactory
-from mock.mock import Mock, patch
 from moto import mock_aws
 from organisations.models import (
     DivisionProblem,