[REFACTOR] Use PEP8 file naming convention (#454)

.gitignore

@@ -3,13 +3,15 @@ __pycache__/
 .vscode/
 .ruff_cache/
 .mypy_cache
-
-.env
-local-tmp/
+.DS_Store
 
 pyproject.toml
 poetry.lock
+.env
 .venv
+.envrc
+
+local-tmp/
 
 data/
 output/

workflows/data_pipelines/agence_bio/README.md

@@ -4,6 +4,7 @@
 
 | Information | Valeur |
 | -------- | -------- |
+| Fichier source     | `dag.py`     |
 | Description | Ce traitement permet de constituer un stock sur les entreprises certifiées bio en France. |
 | Fréquence | Quotidienne |
 | Données sources | [API Professionnels Bio](https://api.gouv.fr/les-api/api-professionnels-bio) |

workflows/data_pipelines/agence_bio/agence_bio_dag.py → workflows/data_pipelines/agence_bio/dag.py

@@ -5,7 +5,7 @@
 
 from dag_datalake_sirene.helpers import Notification
 from dag_datalake_sirene.config import EMAIL_LIST
-from dag_datalake_sirene.workflows.data_pipelines.agence_bio.agence_bio_processor import (
+from dag_datalake_sirene.workflows.data_pipelines.agence_bio.processor import (
     AgenceBioProcessor,
 )
 

workflows/data_pipelines/agence_bio/agence_bio_processor.py → workflows/data_pipelines/agence_bio/processor.py

@@ -6,10 +6,10 @@
 from dag_datalake_sirene.helpers import DataProcessor, Notification
 from dag_datalake_sirene.helpers.minio_helpers import File
 from dag_datalake_sirene.helpers.utils import flatten_object
-from dag_datalake_sirene.workflows.data_pipelines.agence_bio.agence_bio_api import (
+from dag_datalake_sirene.workflows.data_pipelines.agence_bio.api import (
     BioApiClient,
 )
-from dag_datalake_sirene.workflows.data_pipelines.agence_bio.agence_bio_config import (
+from dag_datalake_sirene.workflows.data_pipelines.agence_bio.config import (
     AGENCE_BIO_CONFIG,
 )
 

workflows/data_pipelines/bilans_financiers/README.md

@@ -4,7 +4,7 @@
 
 | Information | Valeur |
 | -------- | -------- |
-| Fichier source     | `bilans_financiers_dag.py`     |
+| Fichier source     | `dag.py`     |
 | Description | Ce traitement permet de synthétiser les données publiées par l'équipe Signaux Faibles et Fiche Commune Entreprise. |
 | Fréquence | Quotidienne |
 | Données sources | [Jeu de données INPI ratios financiers](https://www.data.gouv.fr/fr/datasets/63cb2e29b22886911440440d/) |

workflows/data_pipelines/bilans_financiers/bilans_financiers_dag.py → workflows/data_pipelines/bilans_financiers/dag.py

@@ -5,7 +5,7 @@
 
 from dag_datalake_sirene.helpers import Notification
 from dag_datalake_sirene.config import EMAIL_LIST
-from dag_datalake_sirene.workflows.data_pipelines.bilans_financiers.bilans_financiers_processor import (
+from dag_datalake_sirene.workflows.data_pipelines.bilans_financiers.processor import (
     BilansFinanciersProcessor,
 )
 

workflows/data_pipelines/bilans_financiers/bilans_financiers_processor.py → workflows/data_pipelines/bilans_financiers/processor.py

@@ -4,7 +4,7 @@
 
 from dag_datalake_sirene.helpers import DataProcessor, Notification
 from dag_datalake_sirene.helpers.utils import get_fiscal_year
-from dag_datalake_sirene.workflows.data_pipelines.bilans_financiers.bilans_financiers_config import (
+from dag_datalake_sirene.workflows.data_pipelines.bilans_financiers.config import (
     BILANS_FINANCIERS_CONFIG,
 )
 

workflows/data_pipelines/colter/colter_dag.py → workflows/data_pipelines/colter/dag.py

@@ -6,11 +6,11 @@
 
 from dag_datalake_sirene.config import EMAIL_LIST
 from dag_datalake_sirene.helpers import Notification
-from dag_datalake_sirene.workflows.data_pipelines.colter.colter_config import (
+from dag_datalake_sirene.workflows.data_pipelines.colter.config import (
     COLTER_CONFIG,
     ELUS_CONFIG,
 )
-from dag_datalake_sirene.workflows.data_pipelines.colter.colter_processor import (
+from dag_datalake_sirene.workflows.data_pipelines.colter.processor import (
     ColterProcessor,
     ElusProcessor,
 )

workflows/data_pipelines/colter/colter_processor.py → workflows/data_pipelines/colter/processor.py

@@ -6,7 +6,7 @@
 
 from dag_datalake_sirene.helpers import DataProcessor
 from dag_datalake_sirene.helpers.utils import get_current_year
-from dag_datalake_sirene.workflows.data_pipelines.colter.colter_config import (
+from dag_datalake_sirene.workflows.data_pipelines.colter.config import (
     COLTER_CONFIG,
     ELUS_CONFIG,
 )

workflows/data_pipelines/egapro/DAG.py → workflows/data_pipelines/egapro/dag.py

@@ -3,7 +3,7 @@
 from datetime import timedelta
 from dag_datalake_sirene.config import EMAIL_LIST
 from dag_datalake_sirene.helpers import Notification
-from dag_datalake_sirene.workflows.data_pipelines.egapro.egapro_processor import (
+from dag_datalake_sirene.workflows.data_pipelines.egapro.processor import (
     EgaproProcessor,
 )
 from dag_datalake_sirene.workflows.data_pipelines.egapro.config import EGAPRO_CONFIG

workflows/data_pipelines/ess_france/DAG.py → workflows/data_pipelines/ess_france/dag.py

@@ -3,7 +3,7 @@
 from datetime import timedelta
 from dag_datalake_sirene.config import EMAIL_LIST
 from dag_datalake_sirene.helpers import Notification
-from dag_datalake_sirene.workflows.data_pipelines.ess_france.ess_processor import (
+from dag_datalake_sirene.workflows.data_pipelines.ess_france.processor import (
     EssFranceProcessor,
 )
 from dag_datalake_sirene.workflows.data_pipelines.ess_france.config import ESS_CONFIG

workflows/data_pipelines/etl/data_fetch_clean/agence_bio.py

@@ -1,5 +1,5 @@
 import pandas as pd
-from dag_datalake_sirene.workflows.data_pipelines.agence_bio.agence_bio_config import (
+from dag_datalake_sirene.workflows.data_pipelines.agence_bio.config import (
     AGENCE_BIO_CONFIG,
 )
 

workflows/data_pipelines/etl/data_fetch_clean/bilan_financier.py

@@ -1,5 +1,5 @@
 import pandas as pd
-from dag_datalake_sirene.workflows.data_pipelines.bilans_financiers.bilans_financiers_config import (
+from dag_datalake_sirene.workflows.data_pipelines.bilans_financiers.config import (
     BILANS_FINANCIERS_CONFIG,
 )
 

workflows/data_pipelines/etl/data_fetch_clean/collectivite_territoriale.py

@@ -1,6 +1,6 @@
 import pandas as pd
 
-from dag_datalake_sirene.workflows.data_pipelines.colter.colter_config import (
+from dag_datalake_sirene.workflows.data_pipelines.colter.config import (
     COLTER_CONFIG,
     ELUS_CONFIG,
 )

workflows/data_pipelines/etl/data_fetch_clean/entrepreneur_spectacle.py

@@ -1,6 +1,6 @@
 import pandas as pd
 
-from dag_datalake_sirene.workflows.data_pipelines.spectacle.spectacle_config import (
+from dag_datalake_sirene.workflows.data_pipelines.spectacle.config import (
     SPECTACLE_CONFIG,
 )
 

workflows/data_pipelines/etl/data_fetch_clean/organisme_formation.py

@@ -1,6 +1,6 @@
 import pandas as pd
 
-from dag_datalake_sirene.workflows.data_pipelines.formation.formation_config import (
+from dag_datalake_sirene.workflows.data_pipelines.formation.config import (
     FORMATION_CONFIG,
 )
 

workflows/data_pipelines/etl/data_fetch_clean/uai.py

@@ -1,5 +1,5 @@
 import pandas as pd
-from dag_datalake_sirene.workflows.data_pipelines.uai.uai_config import UAI_CONFIG
+from dag_datalake_sirene.workflows.data_pipelines.uai.config import UAI_CONFIG
 
 
 def preprocess_uai_data(data_dir):

workflows/data_pipelines/etl/task_functions/create_json_last_modified.py

@@ -10,10 +10,10 @@
 )
 from dag_datalake_sirene.workflows.data_pipelines.egapro.config import EGAPRO_CONFIG
 from dag_datalake_sirene.workflows.data_pipelines.finess.config import FINESS_CONFIG
-from dag_datalake_sirene.workflows.data_pipelines.agence_bio.agence_bio_config import (
+from dag_datalake_sirene.workflows.data_pipelines.agence_bio.config import (
     AGENCE_BIO_CONFIG,
 )
-from dag_datalake_sirene.workflows.data_pipelines.bilans_financiers.bilans_financiers_config import (
+from dag_datalake_sirene.workflows.data_pipelines.bilans_financiers.config import (
     BILANS_FINANCIERS_CONFIG,
 )
 from dag_datalake_sirene.workflows.data_pipelines.ess_france.config import (
@@ -22,16 +22,16 @@
 from dag_datalake_sirene.workflows.data_pipelines.rge.config import (
     RGE_CONFIG,
 )
-from dag_datalake_sirene.workflows.data_pipelines.spectacle.spectacle_config import (
+from dag_datalake_sirene.workflows.data_pipelines.spectacle.config import (
     SPECTACLE_CONFIG,
 )
-from dag_datalake_sirene.workflows.data_pipelines.formation.formation_config import (
+from dag_datalake_sirene.workflows.data_pipelines.formation.config import (
     FORMATION_CONFIG,
 )
-from dag_datalake_sirene.workflows.data_pipelines.uai.uai_config import (
+from dag_datalake_sirene.workflows.data_pipelines.uai.config import (
     UAI_CONFIG,
 )
-from dag_datalake_sirene.workflows.data_pipelines.colter.colter_config import (
+from dag_datalake_sirene.workflows.data_pipelines.colter.config import (
     COLTER_CONFIG,
     ELUS_CONFIG,
 )

workflows/data_pipelines/finess/DAG.py → workflows/data_pipelines/finess/dag.py

@@ -6,7 +6,7 @@
 from dag_datalake_sirene.config import EMAIL_LIST
 from dag_datalake_sirene.helpers import Notification
 from dag_datalake_sirene.workflows.data_pipelines.finess.config import FINESS_CONFIG
-from dag_datalake_sirene.workflows.data_pipelines.finess.finess_processor import (
+from dag_datalake_sirene.workflows.data_pipelines.finess.processor import (
     FinessProcessor,
 )
 

workflows/data_pipelines/formation/formation_dag.py → workflows/data_pipelines/formation/dag.py

@@ -4,11 +4,11 @@
 from airflow.utils.dates import days_ago
 
 from dag_datalake_sirene.helpers import Notification
-from dag_datalake_sirene.workflows.data_pipelines.formation.formation_config import (
+from dag_datalake_sirene.workflows.data_pipelines.formation.config import (
     FORMATION_CONFIG,
 )
 from dag_datalake_sirene.config import EMAIL_LIST
-from dag_datalake_sirene.workflows.data_pipelines.formation.formation_processor import (
+from dag_datalake_sirene.workflows.data_pipelines.formation.processor import (
     FormationProcessor,
 )
 

workflows/data_pipelines/formation/formation_processor.py → workflows/data_pipelines/formation/processor.py

@@ -3,7 +3,7 @@
 import pandas as pd
 
 from dag_datalake_sirene.helpers import DataProcessor, Notification
-from dag_datalake_sirene.workflows.data_pipelines.formation.formation_config import (
+from dag_datalake_sirene.workflows.data_pipelines.formation.config import (
     FORMATION_CONFIG,
 )
 

workflows/data_pipelines/rge/DAG.py → workflows/data_pipelines/rge/dag.py

@@ -6,11 +6,11 @@
 from dag_datalake_sirene.config import EMAIL_LIST
 from dag_datalake_sirene.helpers import Notification
 from dag_datalake_sirene.workflows.data_pipelines.rge.config import RGE_CONFIG
-from dag_datalake_sirene.workflows.data_pipelines.rge.rge_processor import (
-    RGEProcessor,
+from dag_datalake_sirene.workflows.data_pipelines.rge.processor import (
+    RgeProcessor,
 )
 
-rge_processor = RGEProcessor()
+rge_processor = RgeProcessor()
 default_args = {
     "depends_on_past": False,
     "email_on_failure": True,

workflows/data_pipelines/rge/rge_processor.py → workflows/data_pipelines/rge/processor.py

@@ -7,7 +7,7 @@
 from dag_datalake_sirene.workflows.data_pipelines.rge.config import RGE_CONFIG
 
 
-class RGEProcessor(DataProcessor):
+class RgeProcessor(DataProcessor):
     def __init__(self):
         super().__init__(RGE_CONFIG)
 

workflows/data_pipelines/rne/flux/README.md

@@ -4,7 +4,7 @@
 
 | Information | Valeur |
 | -------- | -------- |
-| Fichier source     | `DAG.py`     |
+| Fichier source     | `dag.py`     |
 | Description | DAG Airflow permettant de récupérer le flux quotidien des données RNE depuis l'API. Le DAG récupère les données des SIREN modifiés depuis la dernière date de modification enregistrée dans les fichiers JSON stockés sur MinIO et jusqu'à la journée précédant l'exécution du DAG. Une fois récupérées, ces données sont sauvegardées sur une instance Minio. Ces fichiers sont par la suite utilisés pour générer le fichier flux dirigeants pour [l'annuaire des entreprises](https://annuaire-entreprises.data.gouv.fr). | Quotidien |
 | Données sources | API RNE INPI |
 | Données de sorties | Fichiers json dans Minio |

workflows/data_pipelines/rne/stock/README.md

@@ -4,7 +4,7 @@
 
 | Information | Valeur |
 | -------- | -------- |
-| Fichier source     | `DAG.py`     |
+| Fichier source     | `dag.py`     |
 | Description | DAG Airflow permettant de récupérer les données stock du RNE (INPI). Le DAG récupère le fichier zip depuis le site de l'INPI, parcours les fichiers json, et les sauvegarde sur une instance Minio. Ces fichiers sont par la suite utilisés pour générer le fichier stock dirigeants pour [l'annuaire des entreprises](https://annuaire-entreprises.data.gouv.fr). | Annuel |
 | Données sources | STOCK RNE INPI |
 | Données de sorties | Fichiers json dans Minio |

workflows/data_pipelines/sirene/flux/DAG.py → workflows/data_pipelines/sirene/flux/dag.py

@@ -5,7 +5,7 @@
 from airflow.decorators import dag, task
 from dag_datalake_sirene.config import EMAIL_LIST
 from dag_datalake_sirene.helpers import Notification
-from dag_datalake_sirene.workflows.data_pipelines.sirene.flux.sirene_flux_processor import (
+from dag_datalake_sirene.workflows.data_pipelines.sirene.flux.processor import (
     SireneFluxProcessor,
 )
 

workflows/data_pipelines/sirene/flux/sirene_flux_processor.py → workflows/data_pipelines/sirene/flux/processor.py

@@ -1,5 +1,5 @@
 from datetime import datetime
-from dag_datalake_sirene.workflows.data_pipelines.sirene.flux.sirene_client import (
+from dag_datalake_sirene.workflows.data_pipelines.sirene.flux.api import (
     SireneApiClient,
 )
 from dag_datalake_sirene.helpers.data_processor import DataProcessor, Notification

workflows/data_pipelines/sirene/stock/DAG.py → workflows/data_pipelines/sirene/stock/dag.py

@@ -5,7 +5,7 @@
 
 from dag_datalake_sirene.config import EMAIL_LIST
 from dag_datalake_sirene.helpers import Notification
-from dag_datalake_sirene.workflows.data_pipelines.sirene.stock.sirene_stock_processor import (
+from dag_datalake_sirene.workflows.data_pipelines.sirene.stock.processor import (
     SireneStockProcessor,
 )
 

workflows/data_pipelines/spectacle/spectacle_dag.py → workflows/data_pipelines/spectacle/dag.py

@@ -4,11 +4,11 @@
 from airflow.utils.dates import days_ago
 
 from dag_datalake_sirene.helpers import Notification
-from dag_datalake_sirene.workflows.data_pipelines.spectacle.spectacle_config import (
+from dag_datalake_sirene.workflows.data_pipelines.spectacle.config import (
     SPECTACLE_CONFIG,
 )
 from dag_datalake_sirene.config import EMAIL_LIST
-from dag_datalake_sirene.workflows.data_pipelines.spectacle.spectacle_processor import (
+from dag_datalake_sirene.workflows.data_pipelines.spectacle.processor import (
     SpectacleProcessor,
 )
 

workflows/data_pipelines/spectacle/spectacle_processor.py → workflows/data_pipelines/spectacle/processor.py

@@ -5,7 +5,7 @@
 
 class SpectacleProcessor(DataProcessor):
     def __init__(self):
-        from dag_datalake_sirene.workflows.data_pipelines.spectacle.spectacle_config import (
+        from dag_datalake_sirene.workflows.data_pipelines.spectacle.config import (
             SPECTACLE_CONFIG,
         )
 

workflows/data_pipelines/uai/README.md

@@ -4,7 +4,7 @@
 
 | Information | Valeur |
 | -------- | -------- |
-| Fichier source     | `uai_dag.py`     |
+| Fichier source     | `dag.py`     |
 | Description | Ce traitement permet de récupérer différentes sources de données autour des établissements scolaires, les traite pour générer un fichier de synthèse complet. |
 | Fréquence | Quotidien |
 | Données sources | [Annuaire de l'éducation du MENJ](https://www.data.gouv.fr/fr/datasets/5889d03fa3a72974cbf0d5b1/)<br />[Principaux établissements d'enseignement supérieur du MESR](https://www.data.gouv.fr/fr/datasets/586dae5ea3a7290df6f4be88/)<br />[Idéo-Structures d'enseignement supérieur de l'ONISEP](https://www.data.gouv.fr/fr/datasets/5fa5e386afdaa6152360f323/) |

workflows/data_pipelines/uai/uai_dag.py → workflows/data_pipelines/uai/dag.py

@@ -5,11 +5,11 @@
 
 from dag_datalake_sirene.config import EMAIL_LIST
 from dag_datalake_sirene.helpers import Notification
-from dag_datalake_sirene.workflows.data_pipelines.uai.uai_config import (
+from dag_datalake_sirene.workflows.data_pipelines.uai.config import (
     UAI_CONFIG,
 )
-from dag_datalake_sirene.workflows.data_pipelines.uai.uai_processor import (
-    UAIProcessor,
+from dag_datalake_sirene.workflows.data_pipelines.uai.processor import (
+    UaiProcessor,
 )
 
 default_args = {
@@ -33,7 +33,7 @@
     on_success_callback=Notification.send_notification_tchap,
 )
 def data_processing_uai():
-    uai_processor = UAIProcessor()
+    uai_processor = UaiProcessor()
 
     @task.bash
     def clean_previous_outputs():

workflows/data_pipelines/uai/uai_processor.py → workflows/data_pipelines/uai/processor.py

@@ -1,12 +1,12 @@
 import pandas as pd
 
 from dag_datalake_sirene.helpers import DataProcessor, Notification
-from dag_datalake_sirene.workflows.data_pipelines.uai.uai_config import (
+from dag_datalake_sirene.workflows.data_pipelines.uai.config import (
     UAI_CONFIG,
 )
 
 
-class UAIProcessor(DataProcessor):
+class UaiProcessor(DataProcessor):
     def __init__(self):
         super().__init__(UAI_CONFIG)