diff --git a/.github/workflows/test_ooniapi_ooniprobe.yml b/.github/workflows/test_ooniapi_ooniprobe.yml new file mode 100644 index 00000000..2504d81b --- /dev/null +++ b/.github/workflows/test_ooniapi_ooniprobe.yml @@ -0,0 +1,25 @@ +name: test ooniapi/ooniprobe +on: push +jobs: + run_tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: 3.11 + + - name: Install hatch + run: pip install hatch + + - name: Run all tests + run: make test-cov + working-directory: ./ooniapi/services/ooniprobe/ + + - name: Upload coverage to codecov + uses: codecov/codecov-action@v3 + with: + flags: ooniprobe + working-directory: ./ooniapi/services/ooniprobe/ diff --git a/ooniapi/services/oonirun/alembic.ini b/ooniapi/common/src/common/alembic.ini similarity index 100% rename from ooniapi/services/oonirun/alembic.ini rename to ooniapi/common/src/common/alembic.ini diff --git a/ooniapi/common/src/common/alembic/README.md b/ooniapi/common/src/common/alembic/README.md new file mode 100644 index 00000000..ebfa9859 --- /dev/null +++ b/ooniapi/common/src/common/alembic/README.md @@ -0,0 +1,18 @@ +## Alembic database migrations + +When you make changes to the DB schema you will have to run the alembic scripts for generating an appropriate migration file. + +This is how you do it: + +1. Create the template migration script + +``` +alembic revision -m "name of the revision" +``` + +2. Edit the newly created python file and fill out the `upgrade()` and `downgrade()` function with the relevant code bits +3. You can now run the migration like so: + +``` +OONI_PG_URL=postgresql://oonipg:oonipg@localhost/oonipg hatch run alembic upgrade head +``` diff --git a/ooniapi/services/oonirun/alembic/__init__.py b/ooniapi/common/src/common/alembic/__init__.py similarity index 100% rename from ooniapi/services/oonirun/alembic/__init__.py rename to ooniapi/common/src/common/alembic/__init__.py diff --git a/ooniapi/services/oonirun/alembic/env.py b/ooniapi/common/src/common/alembic/env.py similarity index 91% rename from ooniapi/services/oonirun/alembic/env.py rename to ooniapi/common/src/common/alembic/env.py index 4b9c0076..0e985afe 100644 --- a/ooniapi/services/oonirun/alembic/env.py +++ b/ooniapi/common/src/common/alembic/env.py @@ -16,13 +16,8 @@ if config.config_file_name is not None: # no cov fileConfig(config.config_file_name) -# add your model's MetaData object here -# for 'autogenerate' support -# from myapp import mymodel -# target_metadata = mymodel.Base.metadata -from oonirun import models - -target_metadata = models.Base.metadata +# we have no interest in 'autogenerate' support +target_metadata = None section = config.config_ini_section config.set_section_option( diff --git a/ooniapi/common/src/common/alembic/script.py.mako b/ooniapi/common/src/common/alembic/script.py.mako new file mode 100644 index 00000000..fbc4b07d --- /dev/null +++ b/ooniapi/common/src/common/alembic/script.py.mako @@ -0,0 +1,26 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/ooniapi/services/oonirun/alembic/versions/981d92cf8790_init_tables.py b/ooniapi/common/src/common/alembic/versions/981d92cf8790_init_tables.py similarity index 100% rename from ooniapi/services/oonirun/alembic/versions/981d92cf8790_init_tables.py rename to ooniapi/common/src/common/alembic/versions/981d92cf8790_init_tables.py diff --git a/ooniapi/common/src/common/alembic/versions/c9119c05cf42_ooniprobe_services.py b/ooniapi/common/src/common/alembic/versions/c9119c05cf42_ooniprobe_services.py new file mode 100644 index 00000000..af86a989 --- /dev/null +++ b/ooniapi/common/src/common/alembic/versions/c9119c05cf42_ooniprobe_services.py @@ -0,0 +1,69 @@ +"""ooniprobe services + +Revision ID: c9119c05cf42 +Revises: 981d92cf8790 +Create Date: 2024-03-22 20:41:51.940695 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.schema import Sequence, CreateSequence + +# revision identifiers, used by Alembic. +revision: str = "c9119c05cf42" +down_revision: Union[str, None] = "981d92cf8790" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + ooniprobe_vpn_provider_id_seq = Sequence("ooniprobe_vpn_provider_id_seq", start=1) + op.execute(CreateSequence(ooniprobe_vpn_provider_id_seq)) + + op.create_table( + "ooniprobe_vpn_provider", + sa.Column( + "id", + sa.String(), + nullable=False, + server_default=ooniprobe_vpn_provider_id_seq.next_value(), + primary_key=True, + ), + sa.Column("date_created", sa.DateTime(timezone=True), nullable=False), + sa.Column("date_updated", sa.DateTime(timezone=True), nullable=False), + sa.Column("provider_name", sa.String(), nullable=False), + sa.Column("openvpn_cert", sa.String(), nullable=True), + sa.Column("openvpn_ca", sa.String(), nullable=False), + sa.Column("openvpn_key", sa.String(), nullable=False), + ) + + ooniprobe_vpn_provider_endpoint_id_seq = Sequence("ooniprobe_vpn_provider_endpoint_id_seq", start=1) + op.execute(CreateSequence(ooniprobe_vpn_provider_endpoint_id_seq)) + + op.create_table( + "ooniprobe_vpn_provider_endpoint", + sa.Column( + "id", + sa.String(), + nullable=False, + server_default=ooniprobe_vpn_provider_endpoint_id_seq.next_value(), + primary_key=True, + ), + sa.Column("date_created", sa.DateTime(timezone=True), nullable=False), + sa.Column("date_updated", sa.DateTime(timezone=True), nullable=False), + sa.Column("address", sa.String(), nullable=False), + sa.Column("protocol", sa.String(), nullable=True), + sa.Column("transport", sa.String(), nullable=True), + sa.Column("provider_id", sa.String(), nullable=False), + sa.ForeignKeyConstraint( + ["provider_id"], + ["ooniprobe_vpn_provider.id"], + ), + ) + +def downgrade() -> None: + op.drop_table("ooniprobe_vpn_provider_endpoint") + op.drop_table("ooniprobe_vpn_provider") diff --git a/ooniapi/common/src/common/config.py b/ooniapi/common/src/common/config.py index 78722aef..8878db7c 100644 --- a/ooniapi/common/src/common/config.py +++ b/ooniapi/common/src/common/config.py @@ -27,3 +27,5 @@ class Settings(BaseSettings): aws_access_key_id: str = "" aws_secret_access_key: str = "" email_source_address: str = "contact+dev@ooni.io" + + vpn_credential_refresh_hours: int = 24 diff --git a/ooniapi/common/src/common/models.py b/ooniapi/common/src/common/models.py new file mode 100644 index 00000000..2f74f40b --- /dev/null +++ b/ooniapi/common/src/common/models.py @@ -0,0 +1,38 @@ +from datetime import datetime, timezone +from typing import List, Dict, Any +from sqlalchemy.types import DateTime, TypeDecorator + + +class UtcDateTime(TypeDecorator): + """ + Taken from: https://github.com/spoqa/sqlalchemy-utc/blob/8409688000ba0f52c928cc38d34069e521c24bae/sqlalchemy_utc/sqltypes.py + Almost equivalent to :class:`~sqlalchemy.types.DateTime` with + ``timezone=True`` option, but it differs from that by: + + - Never silently take naive :class:`~datetime.datetime`, instead it + always raise :exc:`ValueError` unless time zone aware value. + - :class:`~datetime.datetime` value's :attr:`~datetime.datetime.tzinfo` + is always converted to UTC. + - Unlike SQLAlchemy's built-in :class:`~sqlalchemy.types.DateTime`, + it never return naive :class:`~datetime.datetime`, but time zone + aware value, even with SQLite or MySQL. + """ + + impl = DateTime(timezone=True) + cache_ok = True + + def process_bind_param(self, value, dialect): + if value is not None: + if not isinstance(value, datetime): + raise TypeError("expected datetime.datetime, not " + repr(value)) + elif value.tzinfo is None: + raise ValueError("naive datetime is disallowed") + return value.astimezone(timezone.utc) + + def process_result_value(self, value, dialect): + if value is not None: # no cov + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + else: + value = value.astimezone(timezone.utc) + return value diff --git a/ooniapi/services/oonirun/src/oonirun/postgresql.py b/ooniapi/common/src/common/postgresql.py similarity index 100% rename from ooniapi/services/oonirun/src/oonirun/postgresql.py rename to ooniapi/common/src/common/postgresql.py diff --git a/ooniapi/common/src/common/routers.py b/ooniapi/common/src/common/routers.py index 80868140..305d3079 100644 --- a/ooniapi/common/src/common/routers.py +++ b/ooniapi/common/src/common/routers.py @@ -1,5 +1,6 @@ from datetime import date, datetime from pydantic import BaseModel as PydandicBaseModel +from pydantic import ConfigDict ISO_FORMAT_DATETIME = "%Y-%m-%dT%H:%M:%S.%fZ" @@ -7,8 +8,13 @@ class BaseModel(PydandicBaseModel): - class Config: - json_encoders = { + model_config = ConfigDict( + # TODO(art): this should be ported over to the functional serializer + # pattern (https://docs.pydantic.dev/latest/api/functional_serializers/) + # since json_encoders is deprecated, see: + # https://docs.pydantic.dev/2.6/api/config/#pydantic.config.ConfigDict.json_encoders + json_encoders={ datetime: lambda v: v.strftime(ISO_FORMAT_DATETIME), date: lambda v: v.strftime(ISO_FORMAT_DATE), } + ) diff --git a/ooniapi/services/ooniprobe/.dockerignore b/ooniapi/services/ooniprobe/.dockerignore new file mode 100644 index 00000000..4f7a82b5 --- /dev/null +++ b/ooniapi/services/ooniprobe/.dockerignore @@ -0,0 +1,10 @@ +.DS_Store +*.log +*.pyc +*.swp +*.env +.coverage +coverage.xml +dist/ +.venv/ +__pycache__/ diff --git a/ooniapi/services/ooniprobe/.gitignore b/ooniapi/services/ooniprobe/.gitignore new file mode 100644 index 00000000..9a1b4f54 --- /dev/null +++ b/ooniapi/services/ooniprobe/.gitignore @@ -0,0 +1,3 @@ +/dist +/coverage_html +*.coverage* diff --git a/ooniapi/services/ooniprobe/.vscode/settings.json b/ooniapi/services/ooniprobe/.vscode/settings.json new file mode 100644 index 00000000..07ea54eb --- /dev/null +++ b/ooniapi/services/ooniprobe/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "python.defaultInterpreterPath": "${workspaceFolder}/.venv", + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true +} diff --git a/ooniapi/services/ooniprobe/Dockerfile b/ooniapi/services/ooniprobe/Dockerfile new file mode 100644 index 00000000..42fae6d0 --- /dev/null +++ b/ooniapi/services/ooniprobe/Dockerfile @@ -0,0 +1,33 @@ +# Python builder +FROM python:3.11-bookworm as builder +ARG BUILD_LABEL=docker + +WORKDIR /build + +RUN python -m pip install hatch + +COPY . /build + +# When you build stuff on macOS you end up with ._ files +# https://apple.stackexchange.com/questions/14980/why-are-dot-underscore-files-created-and-how-can-i-avoid-them +RUN find /build -type f -name '._*' -delete + +RUN echo "$BUILD_LABEL" > /build/src/ooniprobe/BUILD_LABEL + +RUN hatch build + +### Actual image running on the host +FROM python:3.11-bookworm as runner + +WORKDIR /app + +COPY --from=builder /build/README.md /app/ +COPY --from=builder /build/dist/*.whl /app/ +RUN pip install /app/*whl && rm /app/*whl + +COPY --from=builder /build/src/ooniprobe/common/alembic/ /app/alembic/ +COPY --from=builder /build/src/ooniprobe/common/alembic.ini /app/ +RUN rm -rf /app/alembic/__pycache__ + +CMD ["uvicorn", "ooniprobe.main:app", "--host", "0.0.0.0", "--port", "80"] +EXPOSE 80 diff --git a/ooniapi/services/ooniprobe/LICENSE.txt b/ooniapi/services/ooniprobe/LICENSE.txt new file mode 100644 index 00000000..3ec29c80 --- /dev/null +++ b/ooniapi/services/ooniprobe/LICENSE.txt @@ -0,0 +1,26 @@ +Copyright 2022-present Open Observatory of Network Interference Foundation (OONI) ETS + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/ooniapi/services/ooniprobe/Makefile b/ooniapi/services/ooniprobe/Makefile new file mode 100644 index 00000000..afa98949 --- /dev/null +++ b/ooniapi/services/ooniprobe/Makefile @@ -0,0 +1,64 @@ +SERVICE_NAME ?= ooniprobe + +ECS_CONTAINER_NAME ?= ooniapi-service-$(SERVICE_NAME) +IMAGE_NAME ?= ooni/api-$(SERVICE_NAME) +DATE := $(shell python3 -c "import datetime;print(datetime.datetime.now(datetime.timezone.utc).strftime('%Y%m%d'))") +GIT_FULL_SHA ?= $(shell git rev-parse HEAD) +SHORT_SHA := $(shell echo ${GIT_FULL_SHA} | cut -c1-8) +PKG_VERSION := $(shell hatch version) + +BUILD_LABEL := $(DATE)-$(SHORT_SHA) +VERSION_LABEL = v$(PKG_VERSION) +ENV_LABEL ?= latest + +print-labels: + echo "ECS_CONTAINER_NAME=${ECS_CONTAINER_NAME}" + echo "PKG_VERSION=${PKG_VERSION}" + echo "BUILD_LABEL=${BUILD_LABEL}" + echo "VERSION_LABEL=${VERSION_LABEL}" + echo "ENV_LABEL=${ENV_LABEL}" + +init: + hatch env create + +docker-build: + # We need to use tar -czh to resolve the common dir symlink + tar -czh . | docker build \ + --build-arg BUILD_LABEL=${BUILD_LABEL} \ + -t ${IMAGE_NAME}:${BUILD_LABEL} \ + -t ${IMAGE_NAME}:${VERSION_LABEL} \ + -t ${IMAGE_NAME}:${ENV_LABEL} \ + - + echo "built image: ${IMAGE_NAME}:${BUILD_LABEL} (${IMAGE_NAME}:${VERSION_LABEL} ${IMAGE_NAME}:${ENV_LABEL})" + +docker-push: + # We need to use tar -czh to resolve the common dir symlink + docker push ${IMAGE_NAME}:${BUILD_LABEL} + docker push ${IMAGE_NAME}:${VERSION_LABEL} + docker push ${IMAGE_NAME}:${ENV_LABEL} + +docker-smoketest: + ./scripts/docker-smoketest.sh ${IMAGE_NAME}:${BUILD_LABEL} + +imagedefinitions.json: + echo '[{"name":"${ECS_CONTAINER_NAME}","imageUri":"${IMAGE_NAME}:${BUILD_LABEL}"}]' > imagedefinitions.json + +test: + hatch run test + +test-cov: + hatch run test-cov + +build: + hatch build + +clean: + hatch clean + rm -f imagedefinitions.json + rm -rf build dist *eggs *.egg-info + rm -rf .venv + +run: + hatch run uvicorn $(SERVICE_NAME).main:app + +.PHONY: init test build clean docker print-labels diff --git a/ooniapi/services/ooniprobe/README.md b/ooniapi/services/ooniprobe/README.md new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/ooniapi/services/ooniprobe/README.md @@ -0,0 +1 @@ + diff --git a/ooniapi/services/ooniprobe/buildspec.yml b/ooniapi/services/ooniprobe/buildspec.yml new file mode 100644 index 00000000..07a5d601 --- /dev/null +++ b/ooniapi/services/ooniprobe/buildspec.yml @@ -0,0 +1,29 @@ +version: 0.2 +env: + variables: + OONI_CODE_PATH: ooniapi/services/ooniprobe + DOCKERHUB_SECRET_ID: oonidevops/dockerhub/access_token + +phases: + install: + runtime-versions: + python: 3.11 + + pre_build: + commands: + - echo "Logging in to dockerhub" + - DOCKER_SECRET=$(aws secretsmanager get-secret-value --secret-id $DOCKERHUB_SECRET_ID --query SecretString --output text) + - echo $DOCKER_SECRET | docker login --username ooni --password-stdin + + build: + commands: + - export GIT_FULL_SHA=${CODEBUILD_RESOLVED_SOURCE_VERSION} + - cd $OONI_CODE_PATH + - make docker-build + - make docker-smoketest + - make docker-push + - make imagedefinitions.json + - cat imagedefinitions.json | tee ${CODEBUILD_SRC_DIR}/imagedefinitions.json + +artifacts: + files: imagedefinitions.json diff --git a/ooniapi/services/ooniprobe/pyproject.toml b/ooniapi/services/ooniprobe/pyproject.toml new file mode 100644 index 00000000..ca22cdd2 --- /dev/null +++ b/ooniapi/services/ooniprobe/pyproject.toml @@ -0,0 +1,93 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "ooniprobe" +description = '' +dynamic = ["version"] + +dependencies = [ + "fastapi ~= 0.108.0", + "clickhouse-driver ~= 0.2.6", + "sqlalchemy ~= 2.0.27", + "ujson ~= 5.9.0", + "urllib3 ~= 2.1.0", + "python-dateutil ~= 2.8.2", + "pydantic-settings ~= 2.1.0", + "statsd ~= 4.0.1", + "uvicorn ~= 0.25.0", + "psycopg2 ~= 2.9.9", + "httpx ~= 0.26.0", + "pyjwt ~= 2.8.0", + "alembic ~= 1.13.1", + "prometheus-fastapi-instrumentator ~= 6.1.0", + "prometheus-client", + "pem ~= 23.1.0", +] + +readme = "README.md" +requires-python = ">=3.11" +license = "MPL-2.0" +keywords = [] +authors = [{ name = "OONI", email = "contact@ooni.org" }] +classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", +] + +[project.urls] +Documentation = "https://github.com/ooni/backend" +Issues = "https://github.com/ooni/backend/issues" +Source = "https://github.com/ooni/backend" + +[tool.hatch.version] +path = "src/ooniprobe/__about__.py" + +[tool.hatch.build.targets.sdist] +include = ["BUILD_LABEL"] + +[tool.hatch.build.targets.wheel] +packages = ["src/ooniprobe"] +artifacts = ["BUILD_LABEL"] + +[tool.hatch.envs.default] +dependencies = [ + "pytest", + "pytest-cov", + "click", + "black", + "pytest-postgresql", + "pytest-asyncio", + "freezegun", +] +path = ".venv/" + +[tool.hatch.envs.default.scripts] +test = "pytest {args:tests}" +test-cov = "pytest -s --full-trace --log-level=INFO --log-cli-level=INFO -v --setup-show --cov=./ --cov-report=xml --cov-report=html --cov-report=term {args:tests}" +cov-report = ["coverage report"] +cov = ["test-cov", "cov-report"] + +[tool.pytest.ini_options] +addopts = ["--import-mode=importlib"] + +[tool.coverage.run] +branch = true +parallel = true +source_pkgs = ["ooniprobe", "tests"] +omit = ["src/ooniprobe/common/*", "src/ooniprobe/__about__.py"] + +[tool.coverage.paths] +ooniprobe = ["src/ooniprobe"] +tests = ["tests"] + +[tool.coverage.report] +exclude_lines = ["no cov", "if __name__ == .__main__.:", "if TYPE_CHECKING:"] diff --git a/ooniapi/services/ooniprobe/scripts/docker-smoketest.sh b/ooniapi/services/ooniprobe/scripts/docker-smoketest.sh new file mode 100755 index 00000000..fdc964cc --- /dev/null +++ b/ooniapi/services/ooniprobe/scripts/docker-smoketest.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +set -ex + +if [ $# -eq 0 ]; then + echo "Error: No Docker image name provided." + echo "Usage: $0 [IMAGE_NAME]" + exit 1 +fi + +IMAGE=$1 +CONTAINER_NAME=ooniapi-smoketest-$RANDOM +PORT=$((RANDOM % 10001 + 30000)) + +cleanup() { + echo "cleaning up" + docker logs $CONTAINER_NAME + docker stop $CONTAINER_NAME >/dev/null 2>&1 + docker rm $CONTAINER_NAME >/dev/null 2>&1 +} + +echo "[+] Running smoketest of ${IMAGE}" +docker run -d --name $CONTAINER_NAME -p $PORT:80 ${IMAGE} + +trap cleanup INT TERM EXIT + +sleep 2 +response=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:$PORT/health) +if [ "${response}" -eq 200 ]; then + echo "Smoke test passed: Received 200 OK from /health endpoint." +else + echo "Smoke test failed: Did not receive 200 OK from /health endpoint. Received: $response" + exit 1 +fi diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/__about__.py b/ooniapi/services/ooniprobe/src/ooniprobe/__about__.py new file mode 100644 index 00000000..674608cc --- /dev/null +++ b/ooniapi/services/ooniprobe/src/ooniprobe/__about__.py @@ -0,0 +1 @@ +VERSION = "0.1.0rc0" diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/__init__.py b/ooniapi/services/ooniprobe/src/ooniprobe/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/common b/ooniapi/services/ooniprobe/src/ooniprobe/common new file mode 120000 index 00000000..3f599f25 --- /dev/null +++ b/ooniapi/services/ooniprobe/src/ooniprobe/common @@ -0,0 +1 @@ +../../../../common/src/common \ No newline at end of file diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/dependencies.py b/ooniapi/services/ooniprobe/src/ooniprobe/dependencies.py new file mode 100644 index 00000000..84a8c8a5 --- /dev/null +++ b/ooniapi/services/ooniprobe/src/ooniprobe/dependencies.py @@ -0,0 +1,21 @@ +from functools import lru_cache +from typing import Annotated + +from fastapi import Depends + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from .common.config import Settings +from .common.dependencies import get_settings + + +def get_postgresql_session(settings: Annotated[Settings, Depends(get_settings)]): + engine = create_engine(settings.postgresql_url) + SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) + + db = SessionLocal() + try: + yield db + finally: + db.close() diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/main.py b/ooniapi/services/ooniprobe/src/ooniprobe/main.py new file mode 100644 index 00000000..44a53544 --- /dev/null +++ b/ooniapi/services/ooniprobe/src/ooniprobe/main.py @@ -0,0 +1,97 @@ +import logging +from contextlib import asynccontextmanager + +from fastapi import Depends, FastAPI +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import RedirectResponse + +from pydantic import BaseModel + +from prometheus_fastapi_instrumentator import Instrumentator + +from . import models +from .routers import v2 + +from .dependencies import get_postgresql_session +from .common.dependencies import get_settings +from .common.version import get_build_label +from .common.metrics import mount_metrics +from .__about__ import VERSION + +pkg_name = "ooniprobe" + +build_label = get_build_label(pkg_name) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + settings = get_settings() + logging.basicConfig(level=getattr(logging, settings.log_level.upper())) + mount_metrics(app, instrumentor.registry) + yield + + +app = FastAPI(lifespan=lifespan) + +instrumentor = Instrumentator().instrument( + app, metric_namespace="ooniapi", metric_subsystem="ooniprobe" +) + +# TODO: temporarily enable all +origins = ["*"] +app.add_middleware( + CORSMiddleware, + allow_origins=origins, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +app.include_router(v2.router, prefix="/api") + + +@app.get("/version") +async def version(): + return {"version": VERSION, "build_label": build_label} + + +class HealthStatus(BaseModel): + status: str + errors: list[str] = [] + version: str + build_label: str + + +@app.get("/health") +async def health( + settings=Depends(get_settings), + db=Depends(get_postgresql_session), +): + errors = [] + try: + db.query(models.OONIProbeVPNProvider).limit(1).all() + except Exception as exc: + print(exc) + errors.append("db_error") + + if settings.jwt_encryption_key == "CHANGEME": + errors.append("bad_jwt_secret") + + if settings.prometheus_metrics_password == "CHANGEME": + errors.append("bad_prometheus_password") + + status = "ok" + if len(errors) > 0: + status = "fail" + + return { + "status": status, + "errors": errors, + "version": VERSION, + "build_label": build_label, + } + + +@app.get("/") +async def root(): + return RedirectResponse("/docs") diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/models.py b/ooniapi/services/ooniprobe/src/ooniprobe/models.py new file mode 100644 index 00000000..1cfc2e10 --- /dev/null +++ b/ooniapi/services/ooniprobe/src/ooniprobe/models.py @@ -0,0 +1,51 @@ +from datetime import datetime +from typing import Dict +from .common.models import UtcDateTime +from .common.postgresql import Base +from sqlalchemy import ForeignKey, Sequence, String, Integer +from sqlalchemy.orm import Mapped +from sqlalchemy.orm import mapped_column, relationship + + +class OONIProbeVPNProvider(Base): + __tablename__ = "ooniprobe_vpn_provider" + + id: Mapped[str] = mapped_column( + String, + Sequence("ooniprobe_vpn_provider_id_seq", start=1), + primary_key=True, + nullable=False, + ) + date_updated: Mapped[datetime] = mapped_column(UtcDateTime()) + date_created: Mapped[datetime] = mapped_column(UtcDateTime()) + + provider_name: Mapped[str] = mapped_column() + + openvpn_cert: Mapped[str] = mapped_column(nullable=True) + openvpn_ca: Mapped[str] = mapped_column(nullable=True) + openvpn_key: Mapped[str] = mapped_column(nullable=True) + + endpoints = relationship("OONIProbeVPNProviderEndpoint", back_populates="provider") + + +class OONIProbeVPNProviderEndpoint(Base): + __tablename__ = "ooniprobe_vpn_provider_endpoint" + + id: Mapped[str] = mapped_column( + String, + Sequence("ooniprobe_vpn_provider_endpoint_id_seq", start=1), + primary_key=True, + nullable=False, + ) + date_updated: Mapped[datetime] = mapped_column(UtcDateTime()) + date_created: Mapped[datetime] = mapped_column(UtcDateTime()) + + protocol: Mapped[str] = mapped_column() + address: Mapped[str] = mapped_column() + transport: Mapped[str] = mapped_column() + # TODO: maybe we want this in the future to store location and other + # metadata about an endpoint + # metadata: Mapped[Dict[str, str]] = mapped_column(nullable=True) + + provider_id = mapped_column(ForeignKey("ooniprobe_vpn_provider.id")) + provider = relationship("OONIProbeVPNProvider", back_populates="endpoints") diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v2.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v2.py new file mode 100644 index 00000000..74cd05b7 --- /dev/null +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v2.py @@ -0,0 +1,152 @@ +from datetime import datetime, timedelta, timezone, date +import random +from typing import Dict, List +import logging + +import sqlalchemy as sa +from sqlalchemy.orm import Session +from fastapi import APIRouter, Depends, HTTPException + +from .. import models + +from ..utils import ( + fetch_openvpn_config, + fetch_openvpn_endpoints, + format_endpoint, + upsert_endpoints, +) +from ..common.routers import BaseModel +from ..common.dependencies import get_settings +from ..dependencies import get_postgresql_session + + +log = logging.getLogger(__name__) + +router = APIRouter() + + +class VPNConfig(BaseModel): + provider: str + protocol: str + config: Dict[str, str] + # date_updated is when the credentials or other config has been updated; + # inputs will follow a different lifecycle. + date_updated: str + endpoints: List[str] + + +def update_vpn_provider(db: Session, provider_name: str) -> models.OONIProbeVPNProvider: + """Fetch a fresh config for a given provider and update the database entry""" + # we are only handling a single provider for the time being (riseup). + # TODO: manage an inventory of known providers. + vpn_cert = fetch_openvpn_config() + + try: + provider = ( + db.query(models.OONIProbeVPNProvider) + .filter( + models.OONIProbeVPNProvider.provider_name == provider_name, + ) + .one() + ) + vpn_endpoints = fetch_openvpn_endpoints() + + provider.openvpn_ca = vpn_cert["ca"] + provider.openvpn_cert = vpn_cert["cert"] + provider.openvpn_key = vpn_cert["key"] + provider.date_updated = datetime.now(timezone.utc) + upsert_endpoints(db, vpn_endpoints, provider) + db.commit() + + except sa.orm.exc.NoResultFound: + provider = models.OONIProbeVPNProvider( + provider_name=provider_name, + date_updated=datetime.now(timezone.utc), + date_created=datetime.now(timezone.utc), + openvpn_ca=vpn_cert["ca"], + openvpn_cert=vpn_cert["cert"], + openvpn_key=vpn_cert["key"], + ) + db.add(provider) + vpn_endpoints = fetch_openvpn_endpoints() + upsert_endpoints(db, vpn_endpoints, provider) + db.commit() + + return provider + + +def get_or_update_riseupvpn( + db: Session, provider_name: str, vpn_credential_refresh_hours: int +) -> models.OONIProbeVPNProvider: + """Get a configuration entry for the given provider, or fetch a fresh one if None found""" + provider = ( + db.query(models.OONIProbeVPNProvider) + .filter( + models.OONIProbeVPNProvider.provider_name == provider_name, + models.OONIProbeVPNProvider.date_updated + > datetime.now(timezone.utc) + - timedelta(hours=vpn_credential_refresh_hours), + ) + .first() + ) + if provider: + return provider + + try: + provider = update_vpn_provider(db, provider_name) + return provider + except: + log.error(f"failed to update vpn provider {provider_name}") + + try: + # In this case we at least serve a stale version of the provider instead + # of just failing. + provider = ( + db.query(models.OONIProbeVPNProvider) + .filter( + models.OONIProbeVPNProvider.provider_name == provider_name, + ) + .one() + ) + return provider + except sa.orm.exc.NoResultFound: + raise HTTPException(status_code=500, detail="error updating provider") + + +@router.get("/v2/ooniprobe/vpn-config/{provider_name}", tags=["ooniprobe"]) +def get_vpn_config( + provider_name: str, + db=Depends(get_postgresql_session), + settings=Depends(get_settings), +) -> VPNConfig: + """GET VPN config parameters for a given provider, including authentication""" + log.debug(f"GET vpn config for {provider_name}") + + if provider_name != "riseupvpn": + raise HTTPException(status_code=404, detail="provider not found") + + try: + provider = get_or_update_riseupvpn( + db=db, + provider_name=provider_name, + vpn_credential_refresh_hours=settings.vpn_credential_refresh_hours, + ) + except Exception as exc: + log.error("Error while fetching credentials for riseup: %s", exc) + raise HTTPException(status_code=500, detail="could not fetch credentials") + + endpoints = [ + format_endpoint(provider.provider_name, ep) for ep in provider.endpoints + ] + return VPNConfig( + provider=provider.provider_name, + protocol="openvpn", + config={ + "ca": provider.openvpn_ca, + "cert": provider.openvpn_cert, + "key": provider.openvpn_key, + }, + # Pick 4 random endpoints to serve to the client + endpoints=random.sample(endpoints, min(len(endpoints), 4)), + date_updated=provider.date_updated.strftime("%Y-%m-%dT%H:%M:%S.%fZ"), + ) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/utils.py b/ooniapi/services/ooniprobe/src/ooniprobe/utils.py new file mode 100644 index 00000000..532633a3 --- /dev/null +++ b/ooniapi/services/ooniprobe/src/ooniprobe/utils.py @@ -0,0 +1,96 @@ +""" +VPN Services + +Insert VPN credentials into database. +""" +import base64 +from datetime import datetime, timezone +import itertools +import logging +from typing import Dict, List, Mapping, TypedDict + +from sqlalchemy.orm import Session +import pem +import httpx + +from ooniprobe.models import OONIProbeVPNProvider, OONIProbeVPNProviderEndpoint + +RISEUP_CA_URL = "https://api.black.riseup.net/ca.crt" +RISEUP_CERT_URL = "https://api.black.riseup.net/3/cert" +RISEUP_ENDPOINT_URL = "https://api.black.riseup.net/3/config/eip-service.json" + +log = logging.getLogger(__name__) + + +class OpenVPNConfig(TypedDict): + ca: str + cert: str + key: str + +class OpenVPNEndpoint(TypedDict): + address: str + protocol: str + transport: str + +def fetch_riseup_ca() -> str: + r = httpx.get(RISEUP_CA_URL) + r.raise_for_status() + return r.text.strip() + + +def fetch_riseup_cert() -> str: + r = httpx.get(RISEUP_CERT_URL) + r.raise_for_status() + return r.text.strip() + + +def fetch_openvpn_config() -> OpenVPNConfig: + ca = fetch_riseup_ca() + pem_cert = fetch_riseup_cert() + key, cert = pem.parse(pem_cert) + return OpenVPNConfig(ca=ca, cert=cert.as_text(), key=key.as_text()) + +def fetch_openvpn_endpoints() -> List[OpenVPNEndpoint]: + endpoints = [] + + r = httpx.get(RISEUP_ENDPOINT_URL) + r.raise_for_status() + j = r.json() + for ep in j["gateways"]: + ip = ep["ip_address"] + # TODO(art): do we want to store this metadata somewhere? + #location = ep["location"] + #hostname = ep["host"] + for t in ep["capabilities"]["transport"]: + if t["type"] != "openvpn": + continue + for transport, port in itertools.product(t["protocols"], t["ports"]): + endpoints.append(OpenVPNEndpoint( + address=f"{ip}:{port}", + protocol="openvpn", + transport=transport + )) + return endpoints + +def format_endpoint(provider_name: str, ep: OONIProbeVPNProviderEndpoint) -> str: + return f"{ep.protocol}://{provider_name}.corp/?address={ep.address}&transport={ep.transport}" + +def upsert_endpoints(db: Session, new_endpoints: List[OpenVPNEndpoint], provider: OONIProbeVPNProvider): + new_endpoints_map = {f'{ep["address"]}-{ep["protocol"]}-{ep["transport"]}': ep for ep in new_endpoints} + for endpoint in provider.endpoints: + key = f'{endpoint.address}-{endpoint.protocol}-{endpoint.transport}' + if key in new_endpoints_map: + endpoint.date_updated = datetime.now(timezone.utc) + new_endpoints_map.pop(key) + else: + db.delete(endpoint) + + for ep in new_endpoints_map.values(): + db.add(OONIProbeVPNProviderEndpoint( + date_created=datetime.now(timezone.utc), + date_updated=datetime.now(timezone.utc), + protocol=ep["protocol"], + address=ep["address"], + transport=ep["transport"], + provider=provider + )) \ No newline at end of file diff --git a/ooniapi/services/ooniprobe/tests/__init__.py b/ooniapi/services/ooniprobe/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ooniapi/services/ooniprobe/tests/conftest.py b/ooniapi/services/ooniprobe/tests/conftest.py new file mode 100644 index 00000000..c51b6004 --- /dev/null +++ b/ooniapi/services/ooniprobe/tests/conftest.py @@ -0,0 +1,80 @@ +import pathlib +import pytest + +import time +import jwt + +from fastapi.testclient import TestClient +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from ooniprobe.common.config import Settings +from ooniprobe.common.dependencies import get_settings +from ooniprobe.main import app + + +def make_override_get_settings(**kw): + def override_get_settings(): + return Settings(**kw) + + return override_get_settings + + +@pytest.fixture +def pg_url(postgresql): + return f"postgresql://{postgresql.info.user}:@{postgresql.info.host}:{postgresql.info.port}/{postgresql.info.dbname}" + + +@pytest.fixture +def db(pg_url): + engine = create_engine(pg_url) + SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) + + db = SessionLocal() + try: + yield db + finally: + db.close() + + +@pytest.fixture +def alembic_migration(pg_url): + from alembic import command + from alembic.config import Config + + migrations_path = ( + pathlib.Path(__file__).parent.parent + / "src" + / "ooniprobe" + / "common" + / "alembic" + ).resolve() + + alembic_cfg = Config() + alembic_cfg.set_main_option("script_location", str(migrations_path)) + alembic_cfg.set_main_option("sqlalchemy.url", pg_url) + + command.upgrade(alembic_cfg, "head") + yield pg_url + + +@pytest.fixture +def client_with_bad_settings(): + app.dependency_overrides[get_settings] = make_override_get_settings( + postgresql_url="postgresql://bad:bad@localhost/bad" + ) + + client = TestClient(app) + yield client + + +@pytest.fixture +def client(alembic_migration): + app.dependency_overrides[get_settings] = make_override_get_settings( + postgresql_url=alembic_migration, + jwt_encryption_key="super_secure", + prometheus_metrics_password="super_secure", + ) + + client = TestClient(app) + yield client diff --git a/ooniapi/services/ooniprobe/tests/test_main.py b/ooniapi/services/ooniprobe/tests/test_main.py new file mode 100644 index 00000000..1f88488e --- /dev/null +++ b/ooniapi/services/ooniprobe/tests/test_main.py @@ -0,0 +1,35 @@ +import pytest + +import httpx +from fastapi.testclient import TestClient +from ooniprobe.main import lifespan, app + + +def test_health_good(client): + r = client.get("health") + j = r.json() + assert j["status"] == "ok", j + assert len(j["errors"]) == 0, j + + +def test_health_bad(client_with_bad_settings): + r = client_with_bad_settings.get("health") + j = r.json() + assert j["status"] == "fail", j + assert len(j["errors"]) > 0, j + + +def test_metrics(client): + r = client.get("/metrics") + + +@pytest.mark.asyncio +async def test_lifecycle(): + async with lifespan(app) as ls: + client = TestClient(app) + r = client.get("/metrics") + assert r.status_code == 401 + + auth = httpx.BasicAuth(username="prom", password="super_secure") + r = client.get("/metrics", auth=auth) + assert r.status_code == 200, r.text diff --git a/ooniapi/services/ooniprobe/tests/test_models.py b/ooniapi/services/ooniprobe/tests/test_models.py new file mode 100644 index 00000000..092238b6 --- /dev/null +++ b/ooniapi/services/ooniprobe/tests/test_models.py @@ -0,0 +1,97 @@ +from datetime import datetime, timedelta, timezone +from ooniprobe.models import OONIProbeVPNProvider, OONIProbeVPNProviderEndpoint +from ooniprobe.utils import OpenVPNEndpoint, upsert_endpoints + + +defaultRiseupTargets = [ + "openvpn://riseup.corp/?address=51.15.187.53:1194&transport=tcp", + "openvpn://riseup.corp/?address=51.15.187.53:1194&transport=udp", +] + +def test_create_providers(db, alembic_migration): + provider = OONIProbeVPNProvider( + provider_name="riseupvpn", + date_created=datetime.now(timezone.utc), + date_updated=datetime.now(timezone.utc), + openvpn_cert="OPENVPN_CERT", + openvpn_ca="OPENVPN_CA", + openvpn_key="OPENVPN_KEY" + ) + db.add(provider) + db.add(OONIProbeVPNProviderEndpoint( + date_created=datetime.now(timezone.utc)-timedelta(hours=1), + date_updated=datetime.now(timezone.utc)-timedelta(hours=1), + protocol="openvpn", + address="51.15.187.53:1194", + transport="tcp", + provider=provider + )) + db.add(OONIProbeVPNProviderEndpoint( + date_created=datetime.now(timezone.utc)-timedelta(hours=1), + date_updated=datetime.now(timezone.utc)-timedelta(hours=1), + protocol="openvpn", + address="51.15.187.53:1194", + transport="udp", + provider=provider + )) + db.add(OONIProbeVPNProviderEndpoint( + date_created=datetime.now(timezone.utc)-timedelta(hours=1), + date_updated=datetime.now(timezone.utc)-timedelta(hours=1), + protocol="openvpn", + address="1.1.1.1:1194", + transport="udp", + provider=provider + )) + db.commit() + + all_endpoints = db.query(OONIProbeVPNProviderEndpoint).all() + assert len(all_endpoints) == 3 + addresses = set() + for endpoint in all_endpoints: + addresses.add(endpoint.address) + assert endpoint.protocol + assert endpoint.address + assert endpoint.transport + assert endpoint.provider.provider_name == "riseupvpn" + assert addresses == set(["51.15.187.53:1194", "1.1.1.1:1194"]) + + provider = db.query(OONIProbeVPNProvider).filter( + OONIProbeVPNProvider.provider_name == "riseupvpn", + OONIProbeVPNProvider.date_updated + > datetime.now(timezone.utc) + - timedelta(days=7), + ).one() + assert len(provider.endpoints) == 3 + + new_endpoints = [ + OpenVPNEndpoint( + address="51.15.187.53:1194", + protocol="openvpn", + transport="udp" + ), + OpenVPNEndpoint( + address="51.15.187.53:1194", + protocol="openvpn", + transport="tcp" + ), + OpenVPNEndpoint( + address="3.2.1.3:1194", + protocol="openvpn", + transport="udp" + ), + ] + + upsert_endpoints(db, new_endpoints, provider) + db.commit() + + all_endpoints = db.query(OONIProbeVPNProviderEndpoint).all() + assert len(all_endpoints) == 3 + addresses = set() + for endpoint in all_endpoints: + addresses.add(endpoint.address) + assert endpoint.protocol + assert endpoint.address + assert endpoint.transport + assert endpoint.provider.provider_name == "riseupvpn" + assert endpoint.date_updated > datetime.now(timezone.utc) - timedelta(minutes=1) + assert addresses == set(["51.15.187.53:1194", "3.2.1.3:1194"]) diff --git a/ooniapi/services/ooniprobe/tests/test_v2.py b/ooniapi/services/ooniprobe/tests/test_v2.py new file mode 100644 index 00000000..887f7770 --- /dev/null +++ b/ooniapi/services/ooniprobe/tests/test_v2.py @@ -0,0 +1,118 @@ +""" +Integration test for OONIProbe API +""" + +from datetime import datetime, timedelta, timezone +from unittest.mock import patch + +from httpx import HTTPError +from freezegun import freeze_time +import pytest + +from ooniprobe.utils import OpenVPNConfig +from ooniprobe import models +from ooniprobe.routers import v2 + +DUMMY_VPN_CERT = OpenVPNConfig( + ca="-----BEGIN CERTIFICATE-----\nSAMPLE CERTIFICATE\n-----END CERTIFICATE-----\n", + cert="-----BEGIN CERTIFICATE-----\nSAMPLE CERTIFICATE\n-----END CERTIFICATE-----\n", + key="-----BEGIN RSA PRIVATE KEY-----\nSAMPLE KEY\n-----END RSA PRIVATE KEY-----\n", +) + + +def test_get_version(client): + r = client.get("/version") + j = r.json() + assert "version" in j + assert "build_label" in j + + +def test_get_root(client): + r = client.get("/") + assert r.status_code == 200 + + +def test_get_config(client): + r = client.get("/api/v2/ooniprobe/vpn-config/riseupvpn") + assert r.status_code == 200 + j = r.json() + assert j["provider"] == "riseupvpn" + assert j["protocol"] == "openvpn" + assert j["config"]["cert"].startswith("-----BEGIN CERTIFICATE") + assert j["config"]["ca"].startswith("-----BEGIN CERTIFICATE") + assert j["config"]["key"].startswith("-----BEGIN RSA PRIVATE KEY") + date_updated = j["date_updated"] + + r = client.get("/api/v2/ooniprobe/vpn-config/riseupvpn") + assert r.status_code == 200 + j = r.json() + assert j["date_updated"] == date_updated + + +def test_invalid_provider_name(client, db): + # we probably aren't going to add NSA VPN to our provider list anytime soon :D + r = client.get("/api/v2/ooniprobe/vpn-config/nsavpn") + assert r.status_code != 200 + + +def test_config_updated(client, db): + with freeze_time("1984-01-01"): + provider = models.OONIProbeVPNProvider( + provider_name="riseupvpn", + date_updated=datetime.now(timezone.utc), + date_created=datetime.now(timezone.utc), + openvpn_ca=DUMMY_VPN_CERT["ca"], + openvpn_cert=DUMMY_VPN_CERT["cert"], + openvpn_key=DUMMY_VPN_CERT["key"], + ) + db.add(provider) + db.commit() + + r = client.get("/api/v2/ooniprobe/vpn-config/riseupvpn") + assert r.status_code == 200 + j = r.json() + assert j["provider"] == "riseupvpn" + assert j["protocol"] == "openvpn" + assert j["config"]["cert"] == DUMMY_VPN_CERT["cert"] + assert j["config"]["ca"] == DUMMY_VPN_CERT["ca"] + assert j["config"]["key"] == DUMMY_VPN_CERT["key"] + + # Check to see if the cert got updated + with freeze_time("1984-04-01"): + r = client.get("/api/v2/ooniprobe/vpn-config/riseupvpn") + assert r.status_code == 200 + j = r.json() + assert j["provider"] == "riseupvpn" + assert j["protocol"] == "openvpn" + assert j["config"]["cert"] != DUMMY_VPN_CERT["cert"] + assert j["config"]["ca"] != DUMMY_VPN_CERT["ca"] + assert j["config"]["key"] != DUMMY_VPN_CERT["key"] + assert j["date_updated"].startswith("1984-04-01") + + +@pytest.mark.parametrize("error", [HTTPError, Exception]) +def test_get_config_fails_if_exception_while_fetching_credentials(client, db, error): + # no previous credential; when forcing any exception on the fetch code the http client should get a 500 + with patch.object(v2, "get_or_update_riseupvpn", side_effect=error("err")): + r = client.get("/api/v2/ooniprobe/vpn-config/riseupvpn") + assert r.status_code == 500 + + with patch.object(v2, "update_vpn_provider", side_effect=error("err")): + r = client.get("/api/v2/ooniprobe/vpn-config/riseupvpn") + assert r.status_code == 500 + + # Check that we get stale data if we have it and it's failing to fetch the data + provider = models.OONIProbeVPNProvider( + provider_name="riseupvpn", + date_updated=datetime.now(timezone.utc) - timedelta(days=20), + date_created=datetime.now(timezone.utc) - timedelta(days=20), + openvpn_ca=DUMMY_VPN_CERT["ca"], + openvpn_cert=DUMMY_VPN_CERT["cert"], + openvpn_key=DUMMY_VPN_CERT["key"], + ) + db.add(provider) + db.commit() + + with patch.object(v2, "update_vpn_provider", side_effect=error("err")): + r = client.get("/api/v2/ooniprobe/vpn-config/riseupvpn") + assert r.status_code == 200 diff --git a/ooniapi/services/oonirun/README.md b/ooniapi/services/oonirun/README.md index 6041f8df..8b137891 100644 --- a/ooniapi/services/oonirun/README.md +++ b/ooniapi/services/oonirun/README.md @@ -1,15 +1 @@ -## Alembic database migrations -When you make changes to the DB schema you will have to run the alembic scripts for generating an appropriate migration file. - -This is how you do it: - -1. Create the template migration script -``` -poetry run alembic revision -m "name of the revision" -``` -2. Edit the newly created python file and fill out the `upgrade()` and `downgrade()` function with the relevant code bits -3. You can now run the migration like so: -``` -OONI_PG_URL=postgresql://oonipg:oonipg@localhost/oonipg hatch run alembic upgrade head -``` diff --git a/ooniapi/services/oonirun/src/oonirun/models.py b/ooniapi/services/oonirun/src/oonirun/models.py index 9b055b4a..efadc5fc 100644 --- a/ooniapi/services/oonirun/src/oonirun/models.py +++ b/ooniapi/services/oonirun/src/oonirun/models.py @@ -1,48 +1,13 @@ -from datetime import datetime, timezone +from datetime import datetime from typing import List, Dict, Any import sqlalchemy as sa from sqlalchemy import ForeignKey, Sequence, String from sqlalchemy.orm import Mapped from sqlalchemy.orm import mapped_column from sqlalchemy.orm import relationship -from sqlalchemy.types import DateTime, TypeDecorator -from .postgresql import Base - - -class UtcDateTime(TypeDecorator): - """ - Taken from: https://github.com/spoqa/sqlalchemy-utc/blob/8409688000ba0f52c928cc38d34069e521c24bae/sqlalchemy_utc/sqltypes.py - Almost equivalent to :class:`~sqlalchemy.types.DateTime` with - ``timezone=True`` option, but it differs from that by: - - - Never silently take naive :class:`~datetime.datetime`, instead it - always raise :exc:`ValueError` unless time zone aware value. - - :class:`~datetime.datetime` value's :attr:`~datetime.datetime.tzinfo` - is always converted to UTC. - - Unlike SQLAlchemy's built-in :class:`~sqlalchemy.types.DateTime`, - it never return naive :class:`~datetime.datetime`, but time zone - aware value, even with SQLite or MySQL. - """ - - impl = DateTime(timezone=True) - cache_ok = True - - def process_bind_param(self, value, dialect): - if value is not None: - if not isinstance(value, datetime): - raise TypeError("expected datetime.datetime, not " + repr(value)) - elif value.tzinfo is None: - raise ValueError("naive datetime is disallowed") - return value.astimezone(timezone.utc) - - def process_result_value(self, value, dialect): - if value is not None: # no cov - if value.tzinfo is None: - value = value.replace(tzinfo=timezone.utc) - else: - value = value.astimezone(timezone.utc) - return value +from .common.models import UtcDateTime +from .common.postgresql import Base class OONIRunLink(Base): diff --git a/ooniapi/services/oonirun/tests/conftest.py b/ooniapi/services/oonirun/tests/conftest.py index 3bf337e3..aab2193f 100644 --- a/ooniapi/services/oonirun/tests/conftest.py +++ b/ooniapi/services/oonirun/tests/conftest.py @@ -25,7 +25,9 @@ def alembic_migration(postgresql): db_url = f"postgresql://{postgresql.info.user}:@{postgresql.info.host}:{postgresql.info.port}/{postgresql.info.dbname}" - migrations_path = (pathlib.Path(__file__).parent.parent / "alembic").resolve() + migrations_path = ( + pathlib.Path(__file__).parent.parent / "src" / "oonirun" / "common" / "alembic" + ).resolve() alembic_cfg = Config() alembic_cfg.set_main_option("script_location", str(migrations_path)) diff --git a/ooniapi/services/oonirun/tests/test_database.py b/ooniapi/services/oonirun/tests/test_database.py index 0606a441..722f41a2 100644 --- a/ooniapi/services/oonirun/tests/test_database.py +++ b/ooniapi/services/oonirun/tests/test_database.py @@ -52,7 +52,9 @@ def config_alembic(db_url): from alembic.config import Config - migrations_path = (pathlib.Path(__file__).parent.parent / "alembic").resolve() + migrations_path = ( + pathlib.Path(__file__).parent.parent / "src" / "oonirun" / "common" / "alembic" + ).resolve() alembic_cfg = Config() alembic_cfg.set_main_option("script_location", str(migrations_path))