syncing vscode

MLproject

@@ -4,15 +4,14 @@ conda_env: conda.yml
 entry_points:
   main:
     parameters:
-
       steps:
-        description: Comma-separated list of steps to execute (useful for debugging)
+        description: "Comma-separated list of steps to execute (useful for debugging)"
         type: str
         default: all
-
       hydra_options:
-        description: Other configuration parameters to override
+        description: "Other configuration parameters to override"
         type: str
-        default: ''
+        default: ""
+    command: "python main.py main.steps='{steps}' {hydra_options}"
+
 
-    command: "python main.py main.steps=\\'{steps}\\' $(echo {hydra_options})"

components/conda.yml

@@ -3,4 +3,17 @@ channels:
   - conda-forge
   - defaults
 dependencies:
+<<<<<<< HEAD
   - mlflow=2.8.1
+  - python=3.10
+  - numpy = 1.24
+=======
+  - python=3.10
+  - pyyaml
+  - hydra-core=1.3.2
+  - pip=23.3.1
+  - numpy = 1.24
+  - pip:
+      - mlflow==2.8.1
+      - wandb==0.16.0
+>>>>>>> 57a4b3afcce7caa68c2a4421e4eb0fd0c923ecb9

components/get_data/conda.yml

@@ -6,7 +6,9 @@ dependencies:
   - pip=23.3.1
   - requests=2.24.0
   - pyarrow
+  - numpy = 1.24
+  - python=3.10
   - pip:
       - mlflow==2.8.1
       - wandb==0.16.0
-      - git+https://github.com/udacity/Project-Build-an-ML-Pipeline-Starter.git#egg=wandb-utils&subdirectory=components
+      - git+https://github.com/JLJ55/Project-Build-an-ML-Pipeline-Starter.git#egg=wandb-utils&subdirectory=components

components/test_regression_model/conda.yml

@@ -3,12 +3,12 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - python=3.10.0
+  - python=3.10
   - pip=23.3.1
   - requests=2.24.0
   - scikit-learn=1.3.2
-  - pandas=2.1.3
+  - numpy=1.24
   - pip:
       - mlflow==2.8.1
       - wandb==0.16.0
-      - git+https://github.com/udacity/Project-Build-an-ML-Pipeline-Starter.git#egg=wandb-utils&subdirectory=components
+      - git+https://github.com/JLJ55/Project-Build-an-ML-Pipeline-Starter.git#egg=wandb-utils&subdirectory=components

components/train_val_test_split/conda.yml

@@ -3,11 +3,12 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - python=3.10.0
+  - python=3.10
   - pip=23.3.1
   - requests=2.24.0
   - scikit-learn=1.3.2
+  - numpy=1.24
   - pip:
       - mlflow==2.8.1
       - wandb==0.16.0
-      - git+https://github.com/udacity/Project-Build-an-ML-Pipeline-Starter.git#egg=wandb-utils&subdirectory=components
+      - git+https://github.com/JLJ55/Project-Build-an-ML-Pipeline-Starter.git#egg=wandb-utils&subdirectory=components

components/train_val_test_split/run.py

@@ -69,4 +69,4 @@ def go(args):
 
     args = parser.parse_args()
 
-    go(args)
+    go(args)

conda.yml

@@ -4,9 +4,15 @@ channels:
   - defaults
 dependencies:
   - python=3.10
+  - numpy=1.24
   - pyyaml
   - hydra-core=1.3.2
   - pip=23.3.1
+  - numpy = 1.24
   - pip:
       - mlflow==2.8.1
       - wandb==0.16.0
+      - pyarrow==7.0.0
+
+
+

config.yaml

@@ -1,10 +1,10 @@
 main:
-  components_repository: "https://github.com/udacity/Project-Build-an-ML-Pipeline-Starter.git#components"
+  components_repository: "https://github.com/JLJ55/Project-Build-an-ML-Pipeline-Starter.git#components"
   # All the intermediate files will be copied to this directory at the end of the run.
   # Set this to null if you are running in prod
   project_name: nyc_airbnb
   experiment_name: development
-  steps: all
+  steps: download
 etl:
   sample: "sample1.csv"
   min_price: 10  # dollars
@@ -35,4 +35,4 @@ modeling:
     criterion: squared_error
     max_features: 0.5
     # DO not change the following
-    oob_score: true
+    oob_score: true

main.py

@@ -1,5 +1,4 @@
 import json
-
 import mlflow
 import tempfile
 import os
@@ -13,84 +12,105 @@
     "data_check",
     "data_split",
     "train_random_forest",
-    # NOTE: We do not include this in the steps so it is not run by mistake.
-    # You first need to promote a model export to "prod" before you can run this,
-    # then you need to run this step explicitly
-#    "test_regression_model"
+    "test_regression_model",
 ]
 
-
 # This automatically reads in the configuration
-@hydra.main(config_name='config')
+@hydra.main(config_name="config")
 def go(config: DictConfig):
 
     # Setup the wandb experiment. All runs will be grouped under this name
     os.environ["WANDB_PROJECT"] = config["main"]["project_name"]
     os.environ["WANDB_RUN_GROUP"] = config["main"]["experiment_name"]
 
     # Steps to execute
-    steps_par = config['main']['steps']
+    steps_par = config["main"]["steps"]
     active_steps = steps_par.split(",") if steps_par != "all" else _steps
 
     # Move to a temporary directory
     with tempfile.TemporaryDirectory() as tmp_dir:
 
         if "download" in active_steps:
-            # Download file and load in W&B
             _ = mlflow.run(
                 f"{config['main']['components_repository']}/get_data",
                 "main",
-                version='main',
+                version="main",
                 env_manager="conda",
                 parameters={
                     "sample": config["etl"]["sample"],
                     "artifact_name": "sample.csv",
                     "artifact_type": "raw_data",
-                    "artifact_description": "Raw file as downloaded"
+                    "artifact_description": "Raw file as downloaded",
                 },
             )
 
         if "basic_cleaning" in active_steps:
-            ##################
-            # Implement here #
-            ##################
-            pass
+            _ = mlflow.run(
+                os.path.join(hydra.utils.get_original_cwd(), "src", "basic_cleaning"),
+                "main",
+                parameters={
+                    "input_artifact": "sample.csv:latest",
+                    "output_artifact": "clean_sample.csv",
+                    "output_type": "clean_sample",
+                    "output_description": "Data with outliers and null values removed",
+                    "min_price": config["etl"]["min_price"],
+                    "max_price": config["etl"]["max_price"],
+                },
+            )
 
         if "data_check" in active_steps:
-            ##################
-            # Implement here #
-            ##################
-            pass
+            _ = mlflow.run(
+                os.path.join(hydra.utils.get_original_cwd(), "src", "data_check"),
+                "main",
+                parameters={
+                    "csv": "clean_sample.csv:latest",
+                    "ref": "clean_sample.csv:reference",
+                    "kl_threshold": config["data_check"]["kl_threshold"],
+                    "min_price": config["etl"]["min_price"],
+                    "max_price": config["etl"]["max_price"],
+                },
+            )
 
         if "data_split" in active_steps:
-            ##################
-            # Implement here #
-            ##################
-            pass
+            _ = mlflow.run(
+                f"{config['main']['components_repository']}/train_val_test_split",
+                "main",
+                parameters={
+                    "input": "clean_sample.csv:latest",
+                    "test_size": config["modeling"]["test_size"],
+                    "random_seed": config["modeling"]["random_seed"],
+                    "stratify_by": config["modeling"]["stratify_by"],
+                },
+            )
 
         if "train_random_forest" in active_steps:
-
-            # NOTE: we need to serialize the random forest configuration into JSON
             rf_config = os.path.abspath("rf_config.json")
             with open(rf_config, "w+") as fp:
-                json.dump(dict(config["modeling"]["random_forest"].items()), fp)  # DO NOT TOUCH
-
-            # NOTE: use the rf_config we just created as the rf_config parameter for the train_random_forest
-            # step
-
-            ##################
-            # Implement here #
-            ##################
+                json.dump(dict(config["modeling"]["random_forest"].items()), fp)
 
-            pass
+            _ = mlflow.run(
+                os.path.join(hydra.utils.get_original_cwd(), "src", "train_random_forest"),
+                "main",
+                parameters={
+                    "trainval_artifact": "trainval_data.csv:latest",
+                    "val_size": config["modeling"]["val_size"],
+                    "random_seed": config["modeling"]["random_seed"],
+                    "stratify_by": config["modeling"]["stratify_by"],
+                    "rf_config": rf_config,
+                    "max_tfidf_features": config["modeling"]["max_tfidf_features"],
+                    "output_artifact": "random_forest_export",
+                },
+            )
 
         if "test_regression_model" in active_steps:
-
-            ##################
-            # Implement here #
-            ##################
-
-            pass
+            _ = mlflow.run(
+                os.path.join(hydra.utils.get_original_cwd(), "components", "test_regression_model"),
+                "main",
+                parameters={
+                    "mlflow_model": "random_forest_export:prod",
+                    "test_dataset": "test_data.csv:latest",
+                },
+            )
 
 
 if __name__ == "__main__":

run_mlflow.py

@@ -0,0 +1,2 @@
+import os
+os.system("mlflow run . -P steps=download")