diff --git a/.gitignore b/.gitignore index 206fb9ca..134cc301 100755 --- a/.gitignore +++ b/.gitignore @@ -3,13 +3,15 @@ __pycache__/ .vscode/ .ruff_cache/ .mypy_cache - -.env -local-tmp/ +.DS_Store pyproject.toml poetry.lock +.env .venv +.envrc + +local-tmp/ data/ output/ diff --git a/workflows/data_pipelines/agence_bio/README.md b/workflows/data_pipelines/agence_bio/README.md index 789ad13b..efb18e28 100644 --- a/workflows/data_pipelines/agence_bio/README.md +++ b/workflows/data_pipelines/agence_bio/README.md @@ -4,6 +4,7 @@ | Information | Valeur | | -------- | -------- | +| Fichier source | `dag.py` | | Description | Ce traitement permet de constituer un stock sur les entreprises certifiées bio en France. | | Fréquence | Quotidienne | | Données sources | [API Professionnels Bio](https://api.gouv.fr/les-api/api-professionnels-bio) | diff --git a/workflows/data_pipelines/agence_bio/agence_bio_api.py b/workflows/data_pipelines/agence_bio/api.py similarity index 100% rename from workflows/data_pipelines/agence_bio/agence_bio_api.py rename to workflows/data_pipelines/agence_bio/api.py diff --git a/workflows/data_pipelines/agence_bio/agence_bio_config.py b/workflows/data_pipelines/agence_bio/config.py similarity index 100% rename from workflows/data_pipelines/agence_bio/agence_bio_config.py rename to workflows/data_pipelines/agence_bio/config.py diff --git a/workflows/data_pipelines/agence_bio/agence_bio_dag.py b/workflows/data_pipelines/agence_bio/dag.py similarity index 94% rename from workflows/data_pipelines/agence_bio/agence_bio_dag.py rename to workflows/data_pipelines/agence_bio/dag.py index fedab69c..af1f011d 100644 --- a/workflows/data_pipelines/agence_bio/agence_bio_dag.py +++ b/workflows/data_pipelines/agence_bio/dag.py @@ -5,7 +5,7 @@ from dag_datalake_sirene.helpers import Notification from dag_datalake_sirene.config import EMAIL_LIST -from dag_datalake_sirene.workflows.data_pipelines.agence_bio.agence_bio_processor import ( +from dag_datalake_sirene.workflows.data_pipelines.agence_bio.processor import ( AgenceBioProcessor, ) diff --git a/workflows/data_pipelines/agence_bio/agence_bio_processor.py b/workflows/data_pipelines/agence_bio/processor.py similarity index 98% rename from workflows/data_pipelines/agence_bio/agence_bio_processor.py rename to workflows/data_pipelines/agence_bio/processor.py index 6cfc4c24..f9dfbcf7 100644 --- a/workflows/data_pipelines/agence_bio/agence_bio_processor.py +++ b/workflows/data_pipelines/agence_bio/processor.py @@ -6,10 +6,10 @@ from dag_datalake_sirene.helpers import DataProcessor, Notification from dag_datalake_sirene.helpers.minio_helpers import File from dag_datalake_sirene.helpers.utils import flatten_object -from dag_datalake_sirene.workflows.data_pipelines.agence_bio.agence_bio_api import ( +from dag_datalake_sirene.workflows.data_pipelines.agence_bio.api import ( BioApiClient, ) -from dag_datalake_sirene.workflows.data_pipelines.agence_bio.agence_bio_config import ( +from dag_datalake_sirene.workflows.data_pipelines.agence_bio.config import ( AGENCE_BIO_CONFIG, ) diff --git a/workflows/data_pipelines/bilans_financiers/README.md b/workflows/data_pipelines/bilans_financiers/README.md index 760ae237..5bde5819 100644 --- a/workflows/data_pipelines/bilans_financiers/README.md +++ b/workflows/data_pipelines/bilans_financiers/README.md @@ -4,7 +4,7 @@ | Information | Valeur | | -------- | -------- | -| Fichier source | `bilans_financiers_dag.py` | +| Fichier source | `dag.py` | | Description | Ce traitement permet de synthétiser les données publiées par l'équipe Signaux Faibles et Fiche Commune Entreprise. | | Fréquence | Quotidienne | | Données sources | [Jeu de données INPI ratios financiers](https://www.data.gouv.fr/fr/datasets/63cb2e29b22886911440440d/) | diff --git a/workflows/data_pipelines/bilans_financiers/bilans_financiers_config.py b/workflows/data_pipelines/bilans_financiers/config.py similarity index 100% rename from workflows/data_pipelines/bilans_financiers/bilans_financiers_config.py rename to workflows/data_pipelines/bilans_financiers/config.py diff --git a/workflows/data_pipelines/bilans_financiers/bilans_financiers_dag.py b/workflows/data_pipelines/bilans_financiers/dag.py similarity index 97% rename from workflows/data_pipelines/bilans_financiers/bilans_financiers_dag.py rename to workflows/data_pipelines/bilans_financiers/dag.py index 2175e109..2ba2496c 100644 --- a/workflows/data_pipelines/bilans_financiers/bilans_financiers_dag.py +++ b/workflows/data_pipelines/bilans_financiers/dag.py @@ -5,7 +5,7 @@ from dag_datalake_sirene.helpers import Notification from dag_datalake_sirene.config import EMAIL_LIST -from dag_datalake_sirene.workflows.data_pipelines.bilans_financiers.bilans_financiers_processor import ( +from dag_datalake_sirene.workflows.data_pipelines.bilans_financiers.processor import ( BilansFinanciersProcessor, ) diff --git a/workflows/data_pipelines/bilans_financiers/bilans_financiers_processor.py b/workflows/data_pipelines/bilans_financiers/processor.py similarity index 98% rename from workflows/data_pipelines/bilans_financiers/bilans_financiers_processor.py rename to workflows/data_pipelines/bilans_financiers/processor.py index 5ba25878..ee50c428 100644 --- a/workflows/data_pipelines/bilans_financiers/bilans_financiers_processor.py +++ b/workflows/data_pipelines/bilans_financiers/processor.py @@ -4,7 +4,7 @@ from dag_datalake_sirene.helpers import DataProcessor, Notification from dag_datalake_sirene.helpers.utils import get_fiscal_year -from dag_datalake_sirene.workflows.data_pipelines.bilans_financiers.bilans_financiers_config import ( +from dag_datalake_sirene.workflows.data_pipelines.bilans_financiers.config import ( BILANS_FINANCIERS_CONFIG, ) diff --git a/workflows/data_pipelines/colter/colter_config.py b/workflows/data_pipelines/colter/config.py similarity index 100% rename from workflows/data_pipelines/colter/colter_config.py rename to workflows/data_pipelines/colter/config.py diff --git a/workflows/data_pipelines/colter/colter_dag.py b/workflows/data_pipelines/colter/dag.py similarity index 94% rename from workflows/data_pipelines/colter/colter_dag.py rename to workflows/data_pipelines/colter/dag.py index a212d252..114c0e92 100644 --- a/workflows/data_pipelines/colter/colter_dag.py +++ b/workflows/data_pipelines/colter/dag.py @@ -6,11 +6,11 @@ from dag_datalake_sirene.config import EMAIL_LIST from dag_datalake_sirene.helpers import Notification -from dag_datalake_sirene.workflows.data_pipelines.colter.colter_config import ( +from dag_datalake_sirene.workflows.data_pipelines.colter.config import ( COLTER_CONFIG, ELUS_CONFIG, ) -from dag_datalake_sirene.workflows.data_pipelines.colter.colter_processor import ( +from dag_datalake_sirene.workflows.data_pipelines.colter.processor import ( ColterProcessor, ElusProcessor, ) diff --git a/workflows/data_pipelines/colter/colter_processor.py b/workflows/data_pipelines/colter/processor.py similarity index 99% rename from workflows/data_pipelines/colter/colter_processor.py rename to workflows/data_pipelines/colter/processor.py index 30aaa6ea..a3e42da7 100644 --- a/workflows/data_pipelines/colter/colter_processor.py +++ b/workflows/data_pipelines/colter/processor.py @@ -6,7 +6,7 @@ from dag_datalake_sirene.helpers import DataProcessor from dag_datalake_sirene.helpers.utils import get_current_year -from dag_datalake_sirene.workflows.data_pipelines.colter.colter_config import ( +from dag_datalake_sirene.workflows.data_pipelines.colter.config import ( COLTER_CONFIG, ELUS_CONFIG, ) diff --git a/workflows/data_pipelines/egapro/DAG.py b/workflows/data_pipelines/egapro/dag.py similarity index 95% rename from workflows/data_pipelines/egapro/DAG.py rename to workflows/data_pipelines/egapro/dag.py index c0193dc3..97d4c8df 100644 --- a/workflows/data_pipelines/egapro/DAG.py +++ b/workflows/data_pipelines/egapro/dag.py @@ -3,7 +3,7 @@ from datetime import timedelta from dag_datalake_sirene.config import EMAIL_LIST from dag_datalake_sirene.helpers import Notification -from dag_datalake_sirene.workflows.data_pipelines.egapro.egapro_processor import ( +from dag_datalake_sirene.workflows.data_pipelines.egapro.processor import ( EgaproProcessor, ) from dag_datalake_sirene.workflows.data_pipelines.egapro.config import EGAPRO_CONFIG diff --git a/workflows/data_pipelines/egapro/egapro_processor.py b/workflows/data_pipelines/egapro/processor.py similarity index 100% rename from workflows/data_pipelines/egapro/egapro_processor.py rename to workflows/data_pipelines/egapro/processor.py diff --git a/workflows/data_pipelines/ess_france/DAG.py b/workflows/data_pipelines/ess_france/dag.py similarity index 95% rename from workflows/data_pipelines/ess_france/DAG.py rename to workflows/data_pipelines/ess_france/dag.py index 906b8e13..fdef13f7 100644 --- a/workflows/data_pipelines/ess_france/DAG.py +++ b/workflows/data_pipelines/ess_france/dag.py @@ -3,7 +3,7 @@ from datetime import timedelta from dag_datalake_sirene.config import EMAIL_LIST from dag_datalake_sirene.helpers import Notification -from dag_datalake_sirene.workflows.data_pipelines.ess_france.ess_processor import ( +from dag_datalake_sirene.workflows.data_pipelines.ess_france.processor import ( EssFranceProcessor, ) from dag_datalake_sirene.workflows.data_pipelines.ess_france.config import ESS_CONFIG diff --git a/workflows/data_pipelines/ess_france/ess_processor.py b/workflows/data_pipelines/ess_france/processor.py similarity index 100% rename from workflows/data_pipelines/ess_france/ess_processor.py rename to workflows/data_pipelines/ess_france/processor.py diff --git a/workflows/data_pipelines/etl/data_fetch_clean/agence_bio.py b/workflows/data_pipelines/etl/data_fetch_clean/agence_bio.py index b339820b..2ee7b234 100644 --- a/workflows/data_pipelines/etl/data_fetch_clean/agence_bio.py +++ b/workflows/data_pipelines/etl/data_fetch_clean/agence_bio.py @@ -1,5 +1,5 @@ import pandas as pd -from dag_datalake_sirene.workflows.data_pipelines.agence_bio.agence_bio_config import ( +from dag_datalake_sirene.workflows.data_pipelines.agence_bio.config import ( AGENCE_BIO_CONFIG, ) diff --git a/workflows/data_pipelines/etl/data_fetch_clean/bilan_financier.py b/workflows/data_pipelines/etl/data_fetch_clean/bilan_financier.py index a01c3eed..420a1462 100644 --- a/workflows/data_pipelines/etl/data_fetch_clean/bilan_financier.py +++ b/workflows/data_pipelines/etl/data_fetch_clean/bilan_financier.py @@ -1,5 +1,5 @@ import pandas as pd -from dag_datalake_sirene.workflows.data_pipelines.bilans_financiers.bilans_financiers_config import ( +from dag_datalake_sirene.workflows.data_pipelines.bilans_financiers.config import ( BILANS_FINANCIERS_CONFIG, ) diff --git a/workflows/data_pipelines/etl/data_fetch_clean/collectivite_territoriale.py b/workflows/data_pipelines/etl/data_fetch_clean/collectivite_territoriale.py index a89ecc4c..aad25244 100644 --- a/workflows/data_pipelines/etl/data_fetch_clean/collectivite_territoriale.py +++ b/workflows/data_pipelines/etl/data_fetch_clean/collectivite_territoriale.py @@ -1,6 +1,6 @@ import pandas as pd -from dag_datalake_sirene.workflows.data_pipelines.colter.colter_config import ( +from dag_datalake_sirene.workflows.data_pipelines.colter.config import ( COLTER_CONFIG, ELUS_CONFIG, ) diff --git a/workflows/data_pipelines/etl/data_fetch_clean/entrepreneur_spectacle.py b/workflows/data_pipelines/etl/data_fetch_clean/entrepreneur_spectacle.py index 3ded3a9d..0ba5e5f7 100644 --- a/workflows/data_pipelines/etl/data_fetch_clean/entrepreneur_spectacle.py +++ b/workflows/data_pipelines/etl/data_fetch_clean/entrepreneur_spectacle.py @@ -1,6 +1,6 @@ import pandas as pd -from dag_datalake_sirene.workflows.data_pipelines.spectacle.spectacle_config import ( +from dag_datalake_sirene.workflows.data_pipelines.spectacle.config import ( SPECTACLE_CONFIG, ) diff --git a/workflows/data_pipelines/etl/data_fetch_clean/organisme_formation.py b/workflows/data_pipelines/etl/data_fetch_clean/organisme_formation.py index a8f63f8e..90c3bf17 100644 --- a/workflows/data_pipelines/etl/data_fetch_clean/organisme_formation.py +++ b/workflows/data_pipelines/etl/data_fetch_clean/organisme_formation.py @@ -1,6 +1,6 @@ import pandas as pd -from dag_datalake_sirene.workflows.data_pipelines.formation.formation_config import ( +from dag_datalake_sirene.workflows.data_pipelines.formation.config import ( FORMATION_CONFIG, ) diff --git a/workflows/data_pipelines/etl/data_fetch_clean/uai.py b/workflows/data_pipelines/etl/data_fetch_clean/uai.py index 1d19ed72..ce2a5f29 100644 --- a/workflows/data_pipelines/etl/data_fetch_clean/uai.py +++ b/workflows/data_pipelines/etl/data_fetch_clean/uai.py @@ -1,5 +1,5 @@ import pandas as pd -from dag_datalake_sirene.workflows.data_pipelines.uai.uai_config import UAI_CONFIG +from dag_datalake_sirene.workflows.data_pipelines.uai.config import UAI_CONFIG def preprocess_uai_data(data_dir): diff --git a/workflows/data_pipelines/etl/task_functions/create_json_last_modified.py b/workflows/data_pipelines/etl/task_functions/create_json_last_modified.py index 0de7164c..8af571d5 100644 --- a/workflows/data_pipelines/etl/task_functions/create_json_last_modified.py +++ b/workflows/data_pipelines/etl/task_functions/create_json_last_modified.py @@ -10,10 +10,10 @@ ) from dag_datalake_sirene.workflows.data_pipelines.egapro.config import EGAPRO_CONFIG from dag_datalake_sirene.workflows.data_pipelines.finess.config import FINESS_CONFIG -from dag_datalake_sirene.workflows.data_pipelines.agence_bio.agence_bio_config import ( +from dag_datalake_sirene.workflows.data_pipelines.agence_bio.config import ( AGENCE_BIO_CONFIG, ) -from dag_datalake_sirene.workflows.data_pipelines.bilans_financiers.bilans_financiers_config import ( +from dag_datalake_sirene.workflows.data_pipelines.bilans_financiers.config import ( BILANS_FINANCIERS_CONFIG, ) from dag_datalake_sirene.workflows.data_pipelines.ess_france.config import ( @@ -22,16 +22,16 @@ from dag_datalake_sirene.workflows.data_pipelines.rge.config import ( RGE_CONFIG, ) -from dag_datalake_sirene.workflows.data_pipelines.spectacle.spectacle_config import ( +from dag_datalake_sirene.workflows.data_pipelines.spectacle.config import ( SPECTACLE_CONFIG, ) -from dag_datalake_sirene.workflows.data_pipelines.formation.formation_config import ( +from dag_datalake_sirene.workflows.data_pipelines.formation.config import ( FORMATION_CONFIG, ) -from dag_datalake_sirene.workflows.data_pipelines.uai.uai_config import ( +from dag_datalake_sirene.workflows.data_pipelines.uai.config import ( UAI_CONFIG, ) -from dag_datalake_sirene.workflows.data_pipelines.colter.colter_config import ( +from dag_datalake_sirene.workflows.data_pipelines.colter.config import ( COLTER_CONFIG, ELUS_CONFIG, ) diff --git a/workflows/data_pipelines/finess/DAG.py b/workflows/data_pipelines/finess/dag.py similarity index 95% rename from workflows/data_pipelines/finess/DAG.py rename to workflows/data_pipelines/finess/dag.py index ffae8aa5..ef8b2409 100644 --- a/workflows/data_pipelines/finess/DAG.py +++ b/workflows/data_pipelines/finess/dag.py @@ -6,7 +6,7 @@ from dag_datalake_sirene.config import EMAIL_LIST from dag_datalake_sirene.helpers import Notification from dag_datalake_sirene.workflows.data_pipelines.finess.config import FINESS_CONFIG -from dag_datalake_sirene.workflows.data_pipelines.finess.finess_processor import ( +from dag_datalake_sirene.workflows.data_pipelines.finess.processor import ( FinessProcessor, ) diff --git a/workflows/data_pipelines/finess/finess_processor.py b/workflows/data_pipelines/finess/processor.py similarity index 100% rename from workflows/data_pipelines/finess/finess_processor.py rename to workflows/data_pipelines/finess/processor.py diff --git a/workflows/data_pipelines/formation/formation_config.py b/workflows/data_pipelines/formation/config.py similarity index 100% rename from workflows/data_pipelines/formation/formation_config.py rename to workflows/data_pipelines/formation/config.py diff --git a/workflows/data_pipelines/formation/formation_dag.py b/workflows/data_pipelines/formation/dag.py similarity index 90% rename from workflows/data_pipelines/formation/formation_dag.py rename to workflows/data_pipelines/formation/dag.py index 250226ff..25529f67 100644 --- a/workflows/data_pipelines/formation/formation_dag.py +++ b/workflows/data_pipelines/formation/dag.py @@ -4,11 +4,11 @@ from airflow.utils.dates import days_ago from dag_datalake_sirene.helpers import Notification -from dag_datalake_sirene.workflows.data_pipelines.formation.formation_config import ( +from dag_datalake_sirene.workflows.data_pipelines.formation.config import ( FORMATION_CONFIG, ) from dag_datalake_sirene.config import EMAIL_LIST -from dag_datalake_sirene.workflows.data_pipelines.formation.formation_processor import ( +from dag_datalake_sirene.workflows.data_pipelines.formation.processor import ( FormationProcessor, ) diff --git a/workflows/data_pipelines/formation/formation_processor.py b/workflows/data_pipelines/formation/processor.py similarity index 94% rename from workflows/data_pipelines/formation/formation_processor.py rename to workflows/data_pipelines/formation/processor.py index a0bc8a94..18a24e93 100644 --- a/workflows/data_pipelines/formation/formation_processor.py +++ b/workflows/data_pipelines/formation/processor.py @@ -3,7 +3,7 @@ import pandas as pd from dag_datalake_sirene.helpers import DataProcessor, Notification -from dag_datalake_sirene.workflows.data_pipelines.formation.formation_config import ( +from dag_datalake_sirene.workflows.data_pipelines.formation.config import ( FORMATION_CONFIG, ) diff --git a/workflows/data_pipelines/metadata/cc/DAG-get-metadata-cc.py b/workflows/data_pipelines/metadata/cc/dag.py similarity index 100% rename from workflows/data_pipelines/metadata/cc/DAG-get-metadata-cc.py rename to workflows/data_pipelines/metadata/cc/dag.py diff --git a/workflows/data_pipelines/rge/DAG.py b/workflows/data_pipelines/rge/dag.py similarity index 92% rename from workflows/data_pipelines/rge/DAG.py rename to workflows/data_pipelines/rge/dag.py index 81469f16..324c884f 100644 --- a/workflows/data_pipelines/rge/DAG.py +++ b/workflows/data_pipelines/rge/dag.py @@ -6,11 +6,11 @@ from dag_datalake_sirene.config import EMAIL_LIST from dag_datalake_sirene.helpers import Notification from dag_datalake_sirene.workflows.data_pipelines.rge.config import RGE_CONFIG -from dag_datalake_sirene.workflows.data_pipelines.rge.rge_processor import ( - RGEProcessor, +from dag_datalake_sirene.workflows.data_pipelines.rge.processor import ( + RgeProcessor, ) -rge_processor = RGEProcessor() +rge_processor = RgeProcessor() default_args = { "depends_on_past": False, "email_on_failure": True, diff --git a/workflows/data_pipelines/rge/rge_processor.py b/workflows/data_pipelines/rge/processor.py similarity index 98% rename from workflows/data_pipelines/rge/rge_processor.py rename to workflows/data_pipelines/rge/processor.py index aa059b2c..4a06c845 100644 --- a/workflows/data_pipelines/rge/rge_processor.py +++ b/workflows/data_pipelines/rge/processor.py @@ -7,7 +7,7 @@ from dag_datalake_sirene.workflows.data_pipelines.rge.config import RGE_CONFIG -class RGEProcessor(DataProcessor): +class RgeProcessor(DataProcessor): def __init__(self): super().__init__(RGE_CONFIG) diff --git a/workflows/data_pipelines/rne/flux/README.md b/workflows/data_pipelines/rne/flux/README.md index cdce7288..130e74e6 100644 --- a/workflows/data_pipelines/rne/flux/README.md +++ b/workflows/data_pipelines/rne/flux/README.md @@ -4,7 +4,7 @@ | Information | Valeur | | -------- | -------- | -| Fichier source | `DAG.py` | +| Fichier source | `dag.py` | | Description | DAG Airflow permettant de récupérer le flux quotidien des données RNE depuis l'API. Le DAG récupère les données des SIREN modifiés depuis la dernière date de modification enregistrée dans les fichiers JSON stockés sur MinIO et jusqu'à la journée précédant l'exécution du DAG. Une fois récupérées, ces données sont sauvegardées sur une instance Minio. Ces fichiers sont par la suite utilisés pour générer le fichier flux dirigeants pour [l'annuaire des entreprises](https://annuaire-entreprises.data.gouv.fr). | Quotidien | | Données sources | API RNE INPI | | Données de sorties | Fichiers json dans Minio | diff --git a/workflows/data_pipelines/rne/stock/README.md b/workflows/data_pipelines/rne/stock/README.md index 624511cc..796af642 100644 --- a/workflows/data_pipelines/rne/stock/README.md +++ b/workflows/data_pipelines/rne/stock/README.md @@ -4,7 +4,7 @@ | Information | Valeur | | -------- | -------- | -| Fichier source | `DAG.py` | +| Fichier source | `dag.py` | | Description | DAG Airflow permettant de récupérer les données stock du RNE (INPI). Le DAG récupère le fichier zip depuis le site de l'INPI, parcours les fichiers json, et les sauvegarde sur une instance Minio. Ces fichiers sont par la suite utilisés pour générer le fichier stock dirigeants pour [l'annuaire des entreprises](https://annuaire-entreprises.data.gouv.fr). | Annuel | | Données sources | STOCK RNE INPI | | Données de sorties | Fichiers json dans Minio | diff --git a/workflows/data_pipelines/sirene/flux/sirene_client.py b/workflows/data_pipelines/sirene/flux/api.py similarity index 100% rename from workflows/data_pipelines/sirene/flux/sirene_client.py rename to workflows/data_pipelines/sirene/flux/api.py diff --git a/workflows/data_pipelines/sirene/flux/DAG.py b/workflows/data_pipelines/sirene/flux/dag.py similarity index 94% rename from workflows/data_pipelines/sirene/flux/DAG.py rename to workflows/data_pipelines/sirene/flux/dag.py index 3813993d..c9187920 100644 --- a/workflows/data_pipelines/sirene/flux/DAG.py +++ b/workflows/data_pipelines/sirene/flux/dag.py @@ -5,7 +5,7 @@ from airflow.decorators import dag, task from dag_datalake_sirene.config import EMAIL_LIST from dag_datalake_sirene.helpers import Notification -from dag_datalake_sirene.workflows.data_pipelines.sirene.flux.sirene_flux_processor import ( +from dag_datalake_sirene.workflows.data_pipelines.sirene.flux.processor import ( SireneFluxProcessor, ) diff --git a/workflows/data_pipelines/sirene/flux/sirene_flux_processor.py b/workflows/data_pipelines/sirene/flux/processor.py similarity index 98% rename from workflows/data_pipelines/sirene/flux/sirene_flux_processor.py rename to workflows/data_pipelines/sirene/flux/processor.py index 53799ff1..ea9a4847 100644 --- a/workflows/data_pipelines/sirene/flux/sirene_flux_processor.py +++ b/workflows/data_pipelines/sirene/flux/processor.py @@ -1,5 +1,5 @@ from datetime import datetime -from dag_datalake_sirene.workflows.data_pipelines.sirene.flux.sirene_client import ( +from dag_datalake_sirene.workflows.data_pipelines.sirene.flux.api import ( SireneApiClient, ) from dag_datalake_sirene.helpers.data_processor import DataProcessor, Notification diff --git a/workflows/data_pipelines/sirene/stock/DAG.py b/workflows/data_pipelines/sirene/stock/dag.py similarity index 93% rename from workflows/data_pipelines/sirene/stock/DAG.py rename to workflows/data_pipelines/sirene/stock/dag.py index 826e036c..79051192 100644 --- a/workflows/data_pipelines/sirene/stock/DAG.py +++ b/workflows/data_pipelines/sirene/stock/dag.py @@ -5,7 +5,7 @@ from dag_datalake_sirene.config import EMAIL_LIST from dag_datalake_sirene.helpers import Notification -from dag_datalake_sirene.workflows.data_pipelines.sirene.stock.sirene_stock_processor import ( +from dag_datalake_sirene.workflows.data_pipelines.sirene.stock.processor import ( SireneStockProcessor, ) diff --git a/workflows/data_pipelines/sirene/stock/sirene_stock_processor.py b/workflows/data_pipelines/sirene/stock/processor.py similarity index 100% rename from workflows/data_pipelines/sirene/stock/sirene_stock_processor.py rename to workflows/data_pipelines/sirene/stock/processor.py diff --git a/workflows/data_pipelines/spectacle/spectacle_config.py b/workflows/data_pipelines/spectacle/config.py similarity index 100% rename from workflows/data_pipelines/spectacle/spectacle_config.py rename to workflows/data_pipelines/spectacle/config.py diff --git a/workflows/data_pipelines/spectacle/spectacle_dag.py b/workflows/data_pipelines/spectacle/dag.py similarity index 90% rename from workflows/data_pipelines/spectacle/spectacle_dag.py rename to workflows/data_pipelines/spectacle/dag.py index a200d043..aa3e3f35 100644 --- a/workflows/data_pipelines/spectacle/spectacle_dag.py +++ b/workflows/data_pipelines/spectacle/dag.py @@ -4,11 +4,11 @@ from airflow.utils.dates import days_ago from dag_datalake_sirene.helpers import Notification -from dag_datalake_sirene.workflows.data_pipelines.spectacle.spectacle_config import ( +from dag_datalake_sirene.workflows.data_pipelines.spectacle.config import ( SPECTACLE_CONFIG, ) from dag_datalake_sirene.config import EMAIL_LIST -from dag_datalake_sirene.workflows.data_pipelines.spectacle.spectacle_processor import ( +from dag_datalake_sirene.workflows.data_pipelines.spectacle.processor import ( SpectacleProcessor, ) diff --git a/workflows/data_pipelines/spectacle/spectacle_processor.py b/workflows/data_pipelines/spectacle/processor.py similarity index 98% rename from workflows/data_pipelines/spectacle/spectacle_processor.py rename to workflows/data_pipelines/spectacle/processor.py index 4d4f506d..ab935f85 100644 --- a/workflows/data_pipelines/spectacle/spectacle_processor.py +++ b/workflows/data_pipelines/spectacle/processor.py @@ -5,7 +5,7 @@ class SpectacleProcessor(DataProcessor): def __init__(self): - from dag_datalake_sirene.workflows.data_pipelines.spectacle.spectacle_config import ( + from dag_datalake_sirene.workflows.data_pipelines.spectacle.config import ( SPECTACLE_CONFIG, ) diff --git a/workflows/data_pipelines/uai/README.md b/workflows/data_pipelines/uai/README.md index 473f3233..38882a84 100644 --- a/workflows/data_pipelines/uai/README.md +++ b/workflows/data_pipelines/uai/README.md @@ -4,7 +4,7 @@ | Information | Valeur | | -------- | -------- | -| Fichier source | `uai_dag.py` | +| Fichier source | `dag.py` | | Description | Ce traitement permet de récupérer différentes sources de données autour des établissements scolaires, les traite pour générer un fichier de synthèse complet. | | Fréquence | Quotidien | | Données sources | [Annuaire de l'éducation du MENJ](https://www.data.gouv.fr/fr/datasets/5889d03fa3a72974cbf0d5b1/)
[Principaux établissements d'enseignement supérieur du MESR](https://www.data.gouv.fr/fr/datasets/586dae5ea3a7290df6f4be88/)
[Idéo-Structures d'enseignement supérieur de l'ONISEP](https://www.data.gouv.fr/fr/datasets/5fa5e386afdaa6152360f323/) | diff --git a/workflows/data_pipelines/uai/uai_config.py b/workflows/data_pipelines/uai/config.py similarity index 100% rename from workflows/data_pipelines/uai/uai_config.py rename to workflows/data_pipelines/uai/config.py diff --git a/workflows/data_pipelines/uai/uai_dag.py b/workflows/data_pipelines/uai/dag.py similarity index 88% rename from workflows/data_pipelines/uai/uai_dag.py rename to workflows/data_pipelines/uai/dag.py index a6a89505..bf2d1ac0 100644 --- a/workflows/data_pipelines/uai/uai_dag.py +++ b/workflows/data_pipelines/uai/dag.py @@ -5,11 +5,11 @@ from dag_datalake_sirene.config import EMAIL_LIST from dag_datalake_sirene.helpers import Notification -from dag_datalake_sirene.workflows.data_pipelines.uai.uai_config import ( +from dag_datalake_sirene.workflows.data_pipelines.uai.config import ( UAI_CONFIG, ) -from dag_datalake_sirene.workflows.data_pipelines.uai.uai_processor import ( - UAIProcessor, +from dag_datalake_sirene.workflows.data_pipelines.uai.processor import ( + UaiProcessor, ) default_args = { @@ -33,7 +33,7 @@ on_success_callback=Notification.send_notification_tchap, ) def data_processing_uai(): - uai_processor = UAIProcessor() + uai_processor = UaiProcessor() @task.bash def clean_previous_outputs(): diff --git a/workflows/data_pipelines/uai/uai_processor.py b/workflows/data_pipelines/uai/processor.py similarity index 96% rename from workflows/data_pipelines/uai/uai_processor.py rename to workflows/data_pipelines/uai/processor.py index a98932a9..0ea6175c 100644 --- a/workflows/data_pipelines/uai/uai_processor.py +++ b/workflows/data_pipelines/uai/processor.py @@ -1,12 +1,12 @@ import pandas as pd from dag_datalake_sirene.helpers import DataProcessor, Notification -from dag_datalake_sirene.workflows.data_pipelines.uai.uai_config import ( +from dag_datalake_sirene.workflows.data_pipelines.uai.config import ( UAI_CONFIG, ) -class UAIProcessor(DataProcessor): +class UaiProcessor(DataProcessor): def __init__(self): super().__init__(UAI_CONFIG)