fix: Change folder structure

PacktPublishing · Aug 9, 2024 · 03291b2 · 03291b2
1 parent 631c9e8
commit 03291b2
Show file tree

Hide file tree

Showing 28 changed files with 25 additions and 9 deletions.
diff --git a/...rfaces/orchestrator/pipelines/__init__.py → pipelines/__init__.py b/...rfaces/orchestrator/pipelines/__init__.py → pipelines/__init__.py
diff --git a/...rchestrator/pipelines/digital_data_etl.py → pipelines/digital_data_etl.py b/...rchestrator/pipelines/digital_data_etl.py → pipelines/digital_data_etl.py
@@ -1,6 +1,6 @@
 from zenml import pipeline
 
-from llm_engineering.interfaces.orchestrator.steps.etl import crawl_links, get_or_create_user
+from steps.etl import crawl_links, get_or_create_user
 
 
 @pipeline

diff --git a/...ator/pipelines/export_artifact_to_json.py → pipelines/export_artifact_to_json.py b/...ator/pipelines/export_artifact_to_json.py → pipelines/export_artifact_to_json.py
@@ -3,7 +3,7 @@
 from zenml import pipeline
 from zenml.client import Client
 
-from llm_engineering.interfaces.orchestrator.steps import export as export_steps
+from steps import export as export_steps
 
 
 @pipeline

diff --git a/...estrator/pipelines/feature_engineering.py → pipelines/feature_engineering.py b/...estrator/pipelines/feature_engineering.py → pipelines/feature_engineering.py
@@ -1,6 +1,6 @@
 from zenml import pipeline
 
-from llm_engineering.interfaces.orchestrator.steps import feature_engineering as fe_steps
+from steps import feature_engineering as fe_steps
 
 
 @pipeline

diff --git a/...r/pipelines/generate_instruct_datasets.py → pipelines/generate_instruct_datasets.py b/...r/pipelines/generate_instruct_datasets.py → pipelines/generate_instruct_datasets.py
@@ -1,6 +1,6 @@
 from zenml import pipeline
 
-from llm_engineering.interfaces.orchestrator.steps import generate_instruct_datasets as cd_steps
+from steps import generate_instruct_datasets as cd_steps
 
 
 @pipeline

diff --git a/...rfaces/orchestrator/pipelines/training.py → pipelines/training.py b/...rfaces/orchestrator/pipelines/training.py → pipelines/training.py
@@ -1,7 +1,7 @@
 from zenml import pipeline
 from zenml.client import Client
 
-from llm_engineering.interfaces.orchestrator.steps import training as training_steps
+from steps import training as training_steps
 
 
 @pipeline

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -21,6 +21,7 @@ numpy = "^1.26.4"
 selenium = "^4.21.0"
 webdriver-manager = "^4.0.1"
 beautifulsoup4 = "^4.12.3"
+html2text = "^2024.2.26"
 jmespath = "^1.0.1"
 chromedriver-autoinstaller = "^0.6.4"
 
@@ -59,7 +60,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poe.tasks]
 # Pipelines
-run-digital-data-etl-alex = "python -m llm_engineering.interfaces.orchestrator.run --run-etl --no-cache --etl-config-filename digital_data_etl_alex_vesa.yaml"
+run-digital-data-etl-alex = "python run.py --run-etl --no-cache --etl-config-filename digital_data_etl_alex_vesa.yaml"
 run-digital-data-etl-maxime = "python -m llm_engineering.interfaces.orchestrator.run --run-etl --no-cache --etl-config-filename digital_data_etl_maxime_labonne.yaml"
 run-digital-data-etl-paul = "python -m llm_engineering.interfaces.orchestrator.run --run-etl --no-cache --etl-config-filename digital_data_etl_paul_iusztin.yaml"
 run-digital-data-etl = [
@@ -93,6 +94,10 @@ local-infrastructure-down = [
     "local-zenml-server-down",
 ]
 
+# ZenML
+set-local-stack = "zenml stack set default"
+set-aws-stack = "zenml stack set aws-stack"
+
 # Docker
 build-docker-image = "docker buildx build --platform linux/amd64 -t llmtwin -f Dockerfile ."
 run-docker-digital-data-etl = "docker run --rm --network host --shm-size=2g --env-file .env llmtwin poetry poe run-digital-data-etl"

diff --git a/...ngineering/interfaces/orchestrator/run.py → run.py b/...ngineering/interfaces/orchestrator/run.py → run.py
@@ -3,7 +3,7 @@
 
 import click
 
-from llm_engineering.interfaces.orchestrator.pipelines import (
+from pipelines import (
     digital_data_etl,
     export_artifact_to_json,
     feature_engineering,
@@ -102,7 +102,8 @@ def main(
     pipeline_args = {
         "enable_cache": not no_cache,
     }
-    root_dir = Path(__file__).resolve().parent.parent.parent.parent
+    # root_dir = Path(__file__).resolve().parent.parent.parent.parent
+    root_dir = Path(__file__).resolve().parent
 
     if run_etl:
         run_args_etl = {}

diff --git a/...interfaces/orchestrator/steps/__init__.py → steps/__init__.py b/...interfaces/orchestrator/steps/__init__.py → steps/__init__.py
diff --git a/...rfaces/orchestrator/steps/etl/__init__.py → steps/etl/__init__.py b/...rfaces/orchestrator/steps/etl/__init__.py → steps/etl/__init__.py
diff --git a/...ces/orchestrator/steps/etl/crawl_links.py → steps/etl/crawl_links.py b/...ces/orchestrator/steps/etl/crawl_links.py → steps/etl/crawl_links.py
diff --git a/...hestrator/steps/etl/get_or_create_user.py → steps/etl/get_or_create_user.py b/...hestrator/steps/etl/get_or_create_user.py → steps/etl/get_or_create_user.py
diff --git a/...ces/orchestrator/steps/export/__init__.py → steps/export/__init__.py b/...ces/orchestrator/steps/export/__init__.py → steps/export/__init__.py
diff --git a/...trator/steps/export/serialize_artifact.py → steps/export/serialize_artifact.py b/...trator/steps/export/serialize_artifact.py → steps/export/serialize_artifact.py
diff --git a/...aces/orchestrator/steps/export/to_json.py → steps/export/to_json.py b/...aces/orchestrator/steps/export/to_json.py → steps/export/to_json.py
diff --git a/...tor/steps/feature_engineering/__init__.py → steps/feature_engineering/__init__.py b/...tor/steps/feature_engineering/__init__.py → steps/feature_engineering/__init__.py
diff --git a/...trator/steps/feature_engineering/clean.py → steps/feature_engineering/clean.py b/...trator/steps/feature_engineering/clean.py → steps/feature_engineering/clean.py
diff --git a/.../feature_engineering/load_to_vector_db.py → .../feature_engineering/load_to_vector_db.py b/.../feature_engineering/load_to_vector_db.py → .../feature_engineering/load_to_vector_db.py
diff --git a/...ature_engineering/query_data_warehouse.py → ...ature_engineering/query_data_warehouse.py b/...ature_engineering/query_data_warehouse.py → ...ature_engineering/query_data_warehouse.py
diff --git a/...estrator/steps/feature_engineering/rag.py → steps/feature_engineering/rag.py b/...estrator/steps/feature_engineering/rag.py → steps/feature_engineering/rag.py
diff --git a/...ps/generate_instruct_datasets/__init__.py → steps/generate_instruct_datasets/__init__.py b/...ps/generate_instruct_datasets/__init__.py → steps/generate_instruct_datasets/__init__.py
diff --git a/...erate_instruct_datasets/create_prompts.py → ...erate_instruct_datasets/create_prompts.py b/...erate_instruct_datasets/create_prompts.py → ...erate_instruct_datasets/create_prompts.py
diff --git a/...ps/generate_instruct_datasets/generate.py → steps/generate_instruct_datasets/generate.py b/...ps/generate_instruct_datasets/generate.py → steps/generate_instruct_datasets/generate.py
diff --git a/..._instruct_datasets/push_to_huggingface.py → ..._instruct_datasets/push_to_huggingface.py b/..._instruct_datasets/push_to_huggingface.py → ..._instruct_datasets/push_to_huggingface.py
diff --git a/..._instruct_datasets/query_feature_store.py → ..._instruct_datasets/query_feature_store.py b/..._instruct_datasets/query_feature_store.py → ..._instruct_datasets/query_feature_store.py
diff --git a/...s/orchestrator/steps/training/__init__.py → steps/training/__init__.py b/...s/orchestrator/steps/training/__init__.py → steps/training/__init__.py
diff --git a/...s/orchestrator/steps/training/tokenize.py → steps/training/tokenize.py b/...s/orchestrator/steps/training/tokenize.py → steps/training/tokenize.py