Merge branch 'main' into optimize-triggers

.docker/docker-compose.yml

@@ -42,11 +42,23 @@ services:
       - "6379:6379"
     volumes:
       - redis_data:/data
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.6.1
+    container_name: elasticsearch
+    restart: always
+    environment:
+      - discovery.type=single-node
+    ports:
+      - "9200:9200"
+      - "9300:9300"
+    volumes:
+      - es_data:/usr/share/elasticsearch/data
 
 volumes:
   data:
     driver: local
     name: zlv-db
   redis_data:
     driver: local
-
+  es_data:
+      driver: local

.eslintignore

@@ -14,6 +14,8 @@ jest.config.js
 package.json
 package-lock.json
 scalingo.json
+yarn.lock
+cron.json
 
 *.css
 *.hbs

.github/workflows/deploy.yml

@@ -15,6 +15,11 @@ on:
         type: string
         required: true
         description: The branch to deploy
+      environment:
+        type: string
+        required: false
+        description: The environment to deploy
+
     secrets:
       CLEVER_SECRET:
         required: true
@@ -29,6 +34,9 @@ on:
 jobs:
   deploy:
     if: ${{ github.ref_name == inputs.branch }}
+    environment:
+      name: ${{ inputs.environment }}
+      url: ${{ vars.HOST }}
     runs-on: ubuntu-latest
     steps:
       - name: Checkout

.github/workflows/e2e.yml

@@ -0,0 +1,49 @@
+name: End-to-end tests
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  e2e:
+    runs-on: ubuntu-latest
+    environment: Staging
+    strategy:
+      matrix:
+        node-version: [20.x]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Enable Corepack
+        run: corepack enable
+
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: yarn
+
+      - run: yarn --version
+
+      - name: Install
+        run: yarn --immutable
+
+      - name: Run cypress tests
+        run: yarn workspace @zerologementvacant/e2e start
+        env:
+          CYPRESS_API: ${{ vars.API_HOST }}
+          CYPRESS_BASE_URL: ${{ vars.HOST }}
+          CYPRESS_EMAIL: ${{ secrets.CYPRESS_EMAIL }}
+          CYPRESS_PASSWORD: ${{ secrets.CYPRESS_PASSWORD }}
+
+      - name: Upload screenshots
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: cypress-screenshots
+          path: e2e/cypress/screenshots
+          retention-days: 7

.github/workflows/github-actions.yml

@@ -59,7 +59,7 @@ jobs:
           PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true
 
       - name: Build
-        run: yarn workspaces foreach --exclude=. --exclude=@zerologementvacant/template -Apt run build
+        run: yarn build
         env:
           CEREMA_TOKEN: token
           DISABLE_ESLINT_PLUGIN: true
@@ -84,6 +84,7 @@ jobs:
       app: Front staging
       alias: front-staging
       branch: main
+      environment: Staging
     secrets: inherit
 
   deploy-api-staging:
@@ -111,6 +112,7 @@ jobs:
       app: Front production
       alias: front-production
       branch: prod
+      environment: Production
     secrets: inherit
 
   deploy-api-production:

.github/workflows/review-app.yml

@@ -34,7 +34,7 @@ jobs:
 
       # TODO: do not create it if exists
       - name: Create postgres addon
-        run: clever addon create postgresql-addon $PR_NAME-postgres --org $ORGA_ID --plan dev --region par
+        run: clever addon create postgresql-addon $PR_NAME-postgres --org $ORGA_ID --plan xxs_sml --region par
 
       - name: Create redis addon
         run: clever addon create redis-addon $PR_NAME-redis --org $ORGA_ID --plan s_mono --region par
@@ -84,6 +84,8 @@ jobs:
           clever env set CEREMA_TOKEN "unused" -a $APP_ALIAS
           clever env set DATABASE_ENV "development" -a $APP_ALIAS
           clever env set DATABASE_URL $POSTGRESQL_ADDON_URI -a $APP_ALIAS
+          clever env set E2E_EMAIL $E2E_EMAIL -a $APP_ALIAS
+          clever env set E2E_PASSWORD $E2E_PASSWORD -a $APP_ALIAS
           clever env set HOST "https://$APP_ALIAS.cleverapps.io" -a $APP_ALIAS
           clever env set METABASE_TOKEN "unused" -a $APP_ALIAS
           clever env set PORT "8080" -a $APP_ALIAS
@@ -105,7 +107,7 @@ jobs:
           clever deploy -a $APP_ALIAS
 
       - name: Scale down for the run
-        run: clever scale --flavor XS -a $APP_ALIAS
+        run: clever scale --flavor S -a $APP_ALIAS
 
   deploy-queue:
     if: github.event.action == 'opened' || github.event.action == 'reopened'
@@ -165,7 +167,7 @@ jobs:
           clever deploy -a $APP_ALIAS
 
       - name: Scale down for the run
-        run: clever scale --flavor XS -a $APP_ALIAS
+        run: clever scale --flavor M -a $APP_ALIAS
 
   deploy-front:
     if: github.event.action == 'opened' || github.event.action == 'reopened'
@@ -250,7 +252,7 @@ jobs:
         run: clever deploy --force -a $APP_ALIAS
 
       - name: Scale down for the run
-        run: clever scale --flavor XS -a $APP_ALIAS
+        run: clever scale --flavor S -a $APP_ALIAS
 
   update-front:
     if: github.event.action == 'synchronize'
@@ -314,7 +316,7 @@ jobs:
         run: clever deploy --force -a $APP_ALIAS
 
       - name: Scale down for the run
-        run: clever scale --flavor XS -a $APP_ALIAS
+        run: clever scale --flavor M -a $APP_ALIAS
 
   delete-queue:
     runs-on: ubuntu-latest

.gitignore

@@ -38,3 +38,5 @@ yarn-error.log*
 .pyc
 __pycache__
 analytics/dagster/storage/
+.duckdb
+.jsonl

.talismanrc

@@ -1,7 +1,19 @@
 fileignoreconfig:
+- filename: .github/workflows/review-app.yml
+  checksum: 78d9abc578bda1e6622a67acfd199ffba381ad98cfeeb5b31bb1fe74c95fc3c1
 - filename: .github/workflows/github-actions.yml
   checksum: b1a53b557b6b2ac4c57f6afce9e902582f5589f97762a963da587496a455c0c1
+- filename: .github/workflows/review-app.yml
+  checksum: 632dba1dc6c13a8f3dd1d8a31fa7be20ceceae67af009c3c7d1f28a64ac05d3f
 - filename: database/*
+- filename: e2e/.env.example
+  checksum: 7afa05e3b64d40c9351b2a60f4ae70887b9c44d66b152c67f7d53b3d4956c93c
+- filename: e2e/README.md
+  checksum: 5f1410c6dd14683c4cb1185cfa8ef4a6018f587abc11b95ea23febcd203d3dee
+- filename: e2e/cypress/e2e/log-in.cy.ts
+  checksum: 5bae716e922600e1360d1b732cd4f0983b2deb3d012ca3bfbf450a4dac02f583
+- filename: e2e/cypress/support/commands.ts
+  checksum: 3bb7cab7d26c40e046724505a5f6dce90a9528f4f626067282ec1c6e3aa9b263
 - filename: frontend/.env.example
   checksum: 9d59fa5d5a2dd989c2aa9a06cf3ccab007df6a002cb963d63406a12ffbcfca8b
 - filename: frontend/src/components/Draft/DraftBody.tsx
@@ -42,12 +54,14 @@ fileignoreconfig:
   checksum: a9dcff1f35295ed617ba50e44cc95c57c63130e80a43e9303c20f160c3b3165d
 - filename: frontend/src/views/Login/LoginView.test.tsx
   checksum: a1fab3ca0b731331cc1e0945e238f60ba319da577710ebeae502813623985a39
+- filename: package.json
+  checksum: bc3388c55b927b286f6e7ec0bf25c060b479eb136dd7a377443c8e99acdf7bd1
 - filename: packages/api-sdk/src/infra/test/token-provider.test.ts
   checksum: c3ea87f28ca17a354e1dadf58e2832088503f258327f4620e0a23a24bb79241a
 - filename: packages/api-sdk/src/infra/token-provider.ts
-  checksum: 0fab9fb960e9b2710b58c4a26737a52f052d36134746f23118e396e22d74dbf1
+  checksum: 033a5387cd4dde54854a07cba893dfa98a493d07fefbc22888f14515e071926d
 - filename: packages/api-sdk/src/sdk.ts
-  checksum: ea407d9dcd6543c66ee0a1c16b794fe5155724f572f8243a26e52b0695bc1fa2
+  checksum: c7c3123970eb5f4347542629da9ac5bcdc3ca31e4828a760121d11cf38d41e39
 - filename: packages/api-sdk/src/test/campaign-api.test.ts
   checksum: e2cabb9dd76c168622f38b4d446907fedcb6f690c86d1c9d1eccbf17a6fe0044
 - filename: packages/api-sdk/src/test/housing-api.test.ts
@@ -56,10 +70,14 @@ fileignoreconfig:
   checksum: 45473f39f4b07b3a33b6138a7e6fc3268690b59439d9f539eb1ce2c1abd430dc
 - filename: packages/schemas/src/password.ts
   checksum: 716d8d646fd8a00af8d4225e15190810f9e8ab082fbb14e7430be109fa676611
+- filename: packages/utils/src/s3.ts
+  checksum: 3270d7c99620a1dd53f1e1a90c24ba12b2b0b090969c498fbe739776f832029a
 - filename: queue/.env.example
   checksum: df71affe33700cb0a36ee0b4824ef60e7d8e5a08f3d4d16ca53c31e9085b45ae
 - filename: queue/README.md
   checksum: fe2ac5277a42e6bf8dfc1e0fe53cc3308bba9019d9091f4b8c291e4a2e4a4eb8
+- filename: queue/src/.htpasswd
+  checksum: 7670652d46bed42247610009a41eadea1d18ff839db9fa831288cb6ab19792e1
 - filename: queue/src/config.ts
   checksum: 8ca4d9a0b6118457fb7dcea24e43645eb63bea92d9e0f1d0c4ae763ea55491a5
 - filename: queue/src/events/index.ts
@@ -69,21 +87,35 @@ fileignoreconfig:
 - filename: queue/src/queue.ts
   checksum: 17c890af8ba6ba90687468438e4615958c5604157ed86904c8c49d6bdaf98ee9
 - filename: queue/src/workers/generate-mails.ts
-  checksum: c781e5dd605469840a3bc8ed26fb7ca72610ae4feea1dabbf36be841412a02a6
+  checksum: 43ae28ea84ab99eab1324356419de4ade2c83b0a70ae999f3d79fea8303a6a3c
+- filename: server/.env.example
+  checksum: 3a2623567475a0868d7cb8a21821b359ebe80fb5cea6bc8559e65cb8756daa26
 - filename: server/controllers/draftController.ts
   checksum: 3627d5cc7172a79998517e1982424462c61116e248e5b6bd29aced81470113e6
 - filename: server/repositories/campaignRepository.ts
   checksum: ab0dcf6bbf41a911aef0755c619c92d18bd04dc651c6ecaf8b1c53ffd5a82309
 - filename: server/services/mailService/brevoService.ts
   checksum: ce84d57e61bfdf4168938e114113005b0961659ea299a3871415ade48ba7b310
 - filename: server/src/controllers/draftController.ts
-  checksum: a169cc6b556976ede5f7ec4933d94cb4dd7d60bf702765443d75cd6736570def
+  checksum: b20a3159d29f2a0964c58d2befc040ef4f553ece130067316e651730c6c97d5b
+- filename: server/src/controllers/fileController.ts
+  checksum: 72d31c3e49e4dbee18222e61e0a5bb48da755cefa2a8d7fa2d9f1a153290279e
+- filename: server/src/controllers/fileRepository.ts
+  checksum: 4387143e5ef9cfb0a7168f63e361fcad10dfef5309e933d9b37c0ce89099d35a
+- filename: server/src/controllers/housingExportController.ts
+  checksum: 41b6d72c503240edea923accce36af8548aa859d3494560eec003bac2c5ab4d5
 - filename: server/src/index.ts
   checksum: 88aec4113a92083b1401a101a673a2cf4457345c258bb8572dccbc713d22a94c
+- filename: server/src/infra/config.ts
+  checksum: 19f56261b2c4fee3c9968934ab851df11ec7c002293a4d1886d7707a373e5e33
+- filename: server/src/infra/database/scripts/001-load-establishments_com_epci_reg_dep.sql
+  checksum: 10a08695a26495460f526ebae824fff99a56afb1f1d548aaff978e01d144f840
 - filename: server/src/infra/database/scripts/002-load-establishments_direction_territoriale.sql
-  checksum: 58d7a51800df5447205a0b9c137478c5bb9d456a76cbc96f0407b9eacc676999
+  checksum: b2db25db2503ffb08d08e232fa5aa319caa0bacca03b5af067038154bb5393ca
 - filename: server/src/infra/database/seeds/development/20240404235457_users.ts
   checksum: f3fabfbe62c13a614236fd7c9474c78de726c44515e8f116f3532765907c218c
+- filename: server/src/infra/database/seeds/development/20240627141242_e2e.ts
+  checksum: 56a46923a8cffefb7e70949b80718dc56ae2f3dd28f5057cb102461fe2fa10eb
 - filename: server/src/routers/unprotected.ts
   checksum: 520455e33ad9672c522364241789eefb4a8d62d86ce90dec626b8a7146ea343e
 - filename: server/src/test/testUtils.ts
@@ -92,6 +124,8 @@ fileignoreconfig:
   checksum: 74df8468c996e2c659a1d697976ea7cf514c4c98b77b6bfca27bd06962e3eed6
 - filename: shared/src/utils/s3.ts
   checksum: 96fa9978f136699921498fe90d02c495d4c1d27f47c96e64b646b219f29204c1
+- filename: yarn.lock
+  checksum: 59d37f8e1ecf81439bce81f285ac09fdcf3978c64ff996c2d0e8731431ecb2c0
 scopeconfig:
 - scope: node
 allowed_patterns:

.yarnrc.yml

@@ -7,6 +7,9 @@ packageExtensions:
   "@turf/dissolve@*":
     dependencies:
       "@turf/flatten": "*"
+  knex@*:
+    dependencies:
+      pg-query-stream: "*"
   postcss-loader@*:
     dependencies:
       browserslist: "*"

analytics/dagster/requirements.txt

@@ -1,3 +1,6 @@
+dagster-dbt
+dbt-duckdb
+
 matplotlib
 pandas
-requests
+requests

analytics/dagster/src/zlv_dbt/pyproject.toml

@@ -0,0 +1,7 @@
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[tool.dagster]
+module_name = "zlv_dbt.definitions"
+code_location_name = "zlv_dbt"

analytics/dagster/src/zlv_dbt/setup.py

@@ -0,0 +1,18 @@
+from setuptools import find_packages, setup
+
+setup(
+    name="zlv_dbt",
+    version="0.0.1",
+    packages=find_packages(),
+    install_requires=[
+        "dagster",
+        "dagster-cloud",
+        "dagster-dbt",
+        "dbt-snowflake<1.9",
+    ],
+    extras_require={
+        "dev": [
+            "dagster-webserver",
+        ]
+    },
+)

analytics/dagster/src/zlv_dbt/zlv_dbt/__init__.py

@@ -0,0 +1 @@
+from .definitions import defs as defs

analytics/dagster/src/zlv_dbt/zlv_dbt/assets.py

@@ -0,0 +1,10 @@
+from dagster import AssetExecutionContext
+from dagster_dbt import DbtCliResource, dbt_assets
+
+from .constants import dbt_manifest_path
+
+
+@dbt_assets(manifest=dbt_manifest_path)
+def zlv_dbt_project_dbt_assets(context: AssetExecutionContext, dbt: DbtCliResource):
+    yield from dbt.cli(["build"], context=context).stream()
+

analytics/dagster/src/zlv_dbt/zlv_dbt/constants.py

@@ -0,0 +1,21 @@
+import os
+from pathlib import Path
+
+from dagster_dbt import DbtCliResource
+
+dbt_project_dir = Path(__file__).joinpath("..", "..", "..", "..", "..", "dbt").resolve()
+dbt = DbtCliResource(project_dir=os.fspath(dbt_project_dir))
+
+# If DAGSTER_DBT_PARSE_PROJECT_ON_LOAD is set, a manifest will be created at run time.
+# Otherwise, we expect a manifest to be present in the project's target directory.
+if os.getenv("DAGSTER_DBT_PARSE_PROJECT_ON_LOAD"):
+    dbt_manifest_path = (
+        dbt.cli(
+            ["--quiet", "parse"],
+            target_path=Path("target"),
+        )
+        .wait()
+        .target_path.joinpath("manifest.json")
+    )
+else:
+    dbt_manifest_path = dbt_project_dir.joinpath("target", "manifest.json")

analytics/dagster/src/zlv_dbt/zlv_dbt/definitions.py

@@ -0,0 +1,16 @@
+import os
+
+from dagster import Definitions
+from dagster_dbt import DbtCliResource
+
+from .assets import zlv_dbt_project_dbt_assets
+from .constants import dbt_project_dir
+from .schedules import schedules
+
+defs = Definitions(
+    assets=[zlv_dbt_project_dbt_assets],
+    schedules=schedules,
+    resources={
+        "dbt": DbtCliResource(project_dir=os.fspath(dbt_project_dir)),
+    },
+)