Feature/riseupvpn config

.github/workflows/test_ooniapi_ooniprobe.yml

@@ -0,0 +1,25 @@
+name: test ooniapi/ooniprobe
+on: push
+jobs:
+  run_tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+
+      - name: Install hatch
+        run: pip install hatch
+
+      - name: Run all tests
+        run: make test-cov
+        working-directory: ./ooniapi/services/ooniprobe/
+
+      - name: Upload coverage to codecov
+        uses: codecov/codecov-action@v3
+        with:
+          flags: ooniprobe
+          working-directory: ./ooniapi/services/ooniprobe/

ooniapi/common/src/common/alembic/README.md

@@ -0,0 +1,18 @@
+## Alembic database migrations
+
+When you make changes to the DB schema you will have to run the alembic scripts for generating an appropriate migration file.
+
+This is how you do it:
+
+1. Create the template migration script
+
+```
+alembic revision -m "name of the revision"
+```
+
+2. Edit the newly created python file and fill out the `upgrade()` and `downgrade()` function with the relevant code bits
+3. You can now run the migration like so:
+
+```
+OONI_PG_URL=postgresql://oonipg:oonipg@localhost/oonipg hatch run alembic upgrade head
+```

ooniapi/services/oonirun/alembic/env.py → ooniapi/common/src/common/alembic/env.py

@@ -16,13 +16,8 @@
 if config.config_file_name is not None:  # no cov
     fileConfig(config.config_file_name)
 
-# add your model's MetaData object here
-# for 'autogenerate' support
-# from myapp import mymodel
-# target_metadata = mymodel.Base.metadata
-from oonirun import models
-
-target_metadata = models.Base.metadata
+# we have no interest in 'autogenerate' support
+target_metadata = None
 
 section = config.config_ini_section
 config.set_section_option(

ooniapi/common/src/common/alembic/script.py.mako

@@ -0,0 +1,26 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}

ooniapi/common/src/common/alembic/versions/c9119c05cf42_ooniprobe_services.py

@@ -0,0 +1,69 @@
+"""ooniprobe services
+
+Revision ID: c9119c05cf42
+Revises: 981d92cf8790
+Create Date: 2024-03-22 20:41:51.940695
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.schema import Sequence, CreateSequence
+
+# revision identifiers, used by Alembic.
+revision: str = "c9119c05cf42"
+down_revision: Union[str, None] = "981d92cf8790"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    ooniprobe_vpn_provider_id_seq = Sequence("ooniprobe_vpn_provider_id_seq", start=1)
+    op.execute(CreateSequence(ooniprobe_vpn_provider_id_seq))
+
+    op.create_table(
+        "ooniprobe_vpn_provider",
+        sa.Column(
+            "id",
+            sa.String(),
+            nullable=False,
+            server_default=ooniprobe_vpn_provider_id_seq.next_value(),
+            primary_key=True,
+        ),
+        sa.Column("date_created", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("date_updated", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("provider_name", sa.String(), nullable=False),
+        sa.Column("openvpn_cert", sa.String(), nullable=True),
+        sa.Column("openvpn_ca", sa.String(), nullable=False),
+        sa.Column("openvpn_key", sa.String(), nullable=False),
+    )
+
+    ooniprobe_vpn_provider_endpoint_id_seq = Sequence("ooniprobe_vpn_provider_endpoint_id_seq", start=1)
+    op.execute(CreateSequence(ooniprobe_vpn_provider_endpoint_id_seq))
+
+    op.create_table(
+        "ooniprobe_vpn_provider_endpoint",
+        sa.Column(
+            "id",
+            sa.String(),
+            nullable=False,
+            server_default=ooniprobe_vpn_provider_endpoint_id_seq.next_value(),
+            primary_key=True,
+        ),
+        sa.Column("date_created", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("date_updated", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("address", sa.String(), nullable=False),
+        sa.Column("protocol", sa.String(), nullable=True),
+        sa.Column("transport", sa.String(), nullable=True),
+        sa.Column("provider_id", sa.String(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["provider_id"],
+            ["ooniprobe_vpn_provider.id"],
+        ),
+    )
+
+def downgrade() -> None:
+    op.drop_table("ooniprobe_vpn_provider_endpoint")
+    op.drop_table("ooniprobe_vpn_provider")

ooniapi/common/src/common/models.py

@@ -0,0 +1,38 @@
+from datetime import datetime, timezone
+from typing import List, Dict, Any
+from sqlalchemy.types import DateTime, TypeDecorator
+
+
+class UtcDateTime(TypeDecorator):
+    """
+    Taken from: https://github.com/spoqa/sqlalchemy-utc/blob/8409688000ba0f52c928cc38d34069e521c24bae/sqlalchemy_utc/sqltypes.py
+    Almost equivalent to :class:`~sqlalchemy.types.DateTime` with
+    ``timezone=True`` option, but it differs from that by:
+
+    - Never silently take naive :class:`~datetime.datetime`, instead it
+      always raise :exc:`ValueError` unless time zone aware value.
+    - :class:`~datetime.datetime` value's :attr:`~datetime.datetime.tzinfo`
+      is always converted to UTC.
+    - Unlike SQLAlchemy's built-in :class:`~sqlalchemy.types.DateTime`,
+      it never return naive :class:`~datetime.datetime`, but time zone
+      aware value, even with SQLite or MySQL.
+    """
+
+    impl = DateTime(timezone=True)
+    cache_ok = True
+
+    def process_bind_param(self, value, dialect):
+        if value is not None:
+            if not isinstance(value, datetime):
+                raise TypeError("expected datetime.datetime, not " + repr(value))
+            elif value.tzinfo is None:
+                raise ValueError("naive datetime is disallowed")
+            return value.astimezone(timezone.utc)
+
+    def process_result_value(self, value, dialect):
+        if value is not None:  # no cov
+            if value.tzinfo is None:
+                value = value.replace(tzinfo=timezone.utc)
+            else:
+                value = value.astimezone(timezone.utc)
+        return value

ooniapi/common/src/common/routers.py

@@ -1,14 +1,20 @@
 from datetime import date, datetime
 from pydantic import BaseModel as PydandicBaseModel
+from pydantic import ConfigDict
 
 
 ISO_FORMAT_DATETIME = "%Y-%m-%dT%H:%M:%S.%fZ"
 ISO_FORMAT_DATE = "%Y-%m-%d"
 
 
 class BaseModel(PydandicBaseModel):
-    class Config:
-        json_encoders = {
+    model_config = ConfigDict(
+        # TODO(art): this should be ported over to the functional serializer
+        # pattern (https://docs.pydantic.dev/latest/api/functional_serializers/)
+        # since json_encoders is deprecated, see:
+        # https://docs.pydantic.dev/2.6/api/config/#pydantic.config.ConfigDict.json_encoders
+        json_encoders={
             datetime: lambda v: v.strftime(ISO_FORMAT_DATETIME),
             date: lambda v: v.strftime(ISO_FORMAT_DATE),
         }
+    )

ooniapi/services/ooniprobe/.dockerignore

@@ -0,0 +1,10 @@
+.DS_Store
+*.log
+*.pyc
+*.swp
+*.env
+.coverage
+coverage.xml
+dist/
+.venv/
+__pycache__/

ooniapi/services/ooniprobe/.gitignore

@@ -0,0 +1,3 @@
+/dist
+/coverage_html
+*.coverage*

ooniapi/services/ooniprobe/.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "python.defaultInterpreterPath": "${workspaceFolder}/.venv",
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestEnabled": true
+}

ooniapi/services/ooniprobe/Dockerfile

@@ -0,0 +1,33 @@
+# Python builder
+FROM python:3.11-bookworm as builder
+ARG BUILD_LABEL=docker
+
+WORKDIR /build
+
+RUN python -m pip install hatch
+
+COPY . /build
+
+# When you build stuff on macOS you end up with ._ files
+# https://apple.stackexchange.com/questions/14980/why-are-dot-underscore-files-created-and-how-can-i-avoid-them
+RUN find /build -type f -name '._*' -delete
+
+RUN echo "$BUILD_LABEL" > /build/src/ooniprobe/BUILD_LABEL
+
+RUN hatch build
+
+### Actual image running on the host
+FROM python:3.11-bookworm as runner
+
+WORKDIR /app
+
+COPY --from=builder /build/README.md /app/
+COPY --from=builder /build/dist/*.whl /app/
+RUN pip install /app/*whl && rm /app/*whl
+
+COPY --from=builder /build/src/ooniprobe/common/alembic/ /app/alembic/
+COPY --from=builder /build/src/ooniprobe/common/alembic.ini /app/
+RUN rm -rf /app/alembic/__pycache__
+
+CMD ["uvicorn", "ooniprobe.main:app", "--host", "0.0.0.0", "--port", "80"]
+EXPOSE 80

ooniapi/services/ooniprobe/LICENSE.txt

@@ -0,0 +1,26 @@
+Copyright 2022-present Open Observatory of Network Interference Foundation (OONI) ETS
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+   may be used to endorse or promote products derived from this software
+   without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

ooniapi/services/ooniprobe/Makefile

@@ -0,0 +1,64 @@
+SERVICE_NAME ?= ooniprobe
+
+ECS_CONTAINER_NAME ?= ooniapi-service-$(SERVICE_NAME)
+IMAGE_NAME ?= ooni/api-$(SERVICE_NAME)
+DATE := $(shell python3 -c "import datetime;print(datetime.datetime.now(datetime.timezone.utc).strftime('%Y%m%d'))")
+GIT_FULL_SHA ?= $(shell git rev-parse HEAD)
+SHORT_SHA := $(shell echo ${GIT_FULL_SHA} | cut -c1-8)
+PKG_VERSION := $(shell hatch version)
+
+BUILD_LABEL := $(DATE)-$(SHORT_SHA)
+VERSION_LABEL = v$(PKG_VERSION)
+ENV_LABEL ?= latest
+
+print-labels:
+	echo "ECS_CONTAINER_NAME=${ECS_CONTAINER_NAME}"
+	echo "PKG_VERSION=${PKG_VERSION}"
+	echo "BUILD_LABEL=${BUILD_LABEL}"
+	echo "VERSION_LABEL=${VERSION_LABEL}"
+	echo "ENV_LABEL=${ENV_LABEL}"
+
+init:
+	hatch env create
+
+docker-build:
+	# We need to use tar -czh to resolve the common dir symlink
+	tar -czh . | docker build \
+		--build-arg BUILD_LABEL=${BUILD_LABEL} \
+		-t ${IMAGE_NAME}:${BUILD_LABEL} \
+		-t ${IMAGE_NAME}:${VERSION_LABEL} \
+		-t ${IMAGE_NAME}:${ENV_LABEL} \
+		-
+	echo "built image: ${IMAGE_NAME}:${BUILD_LABEL} (${IMAGE_NAME}:${VERSION_LABEL} ${IMAGE_NAME}:${ENV_LABEL})"
+
+docker-push:
+	# We need to use tar -czh to resolve the common dir symlink
+	docker push ${IMAGE_NAME}:${BUILD_LABEL}
+	docker push ${IMAGE_NAME}:${VERSION_LABEL}
+	docker push ${IMAGE_NAME}:${ENV_LABEL}
+
+docker-smoketest:
+	./scripts/docker-smoketest.sh ${IMAGE_NAME}:${BUILD_LABEL}
+
+imagedefinitions.json:
+	echo '[{"name":"${ECS_CONTAINER_NAME}","imageUri":"${IMAGE_NAME}:${BUILD_LABEL}"}]' > imagedefinitions.json
+
+test:
+	hatch run test
+
+test-cov:
+	hatch run test-cov
+
+build:
+	hatch build
+
+clean:
+	hatch clean
+	rm -f imagedefinitions.json
+	rm -rf build dist *eggs *.egg-info
+	rm -rf .venv
+
+run:
+	hatch run uvicorn $(SERVICE_NAME).main:app
+
+.PHONY: init test build clean docker print-labels

ooniapi/services/ooniprobe/README.md

@@ -0,0 +1 @@
+

ooniapi/services/ooniprobe/buildspec.yml

@@ -0,0 +1,29 @@
+version: 0.2
+env:
+  variables:
+    OONI_CODE_PATH: ooniapi/services/ooniprobe
+    DOCKERHUB_SECRET_ID: oonidevops/dockerhub/access_token
+
+phases:
+  install:
+    runtime-versions:
+      python: 3.11
+
+  pre_build:
+    commands:
+      - echo "Logging in to dockerhub"
+      - DOCKER_SECRET=$(aws secretsmanager get-secret-value --secret-id $DOCKERHUB_SECRET_ID --query SecretString --output text)
+      - echo $DOCKER_SECRET | docker login --username ooni --password-stdin
+
+  build:
+    commands:
+      - export GIT_FULL_SHA=${CODEBUILD_RESOLVED_SOURCE_VERSION}
+      - cd $OONI_CODE_PATH
+      - make docker-build
+      - make docker-smoketest
+      - make docker-push
+      - make imagedefinitions.json
+      - cat imagedefinitions.json | tee ${CODEBUILD_SRC_DIR}/imagedefinitions.json
+
+artifacts:
+  files: imagedefinitions.json