feat: Simplification of Document Search Evaluation interface

examples/evaluation/document-search/config/optimization.yaml → examples/evaluation/document-search/advanced/config/optimization.yaml

@@ -19,8 +19,9 @@ metrics:
       threshold: 0.5
 
 
-callbacks:
-  - type: ragbits.evaluate.callbacks.neptune:NeptuneCallbackConfigurator
-    args:
-      callback_type: neptune.integrations.optuna:NeptuneCallback
-      project: deepsense-ai/ragbits
+#uncomment to use neptune tracking (set your project name)
+#callbacks:
+#  - type: ragbits.evaluate.callbacks.neptune:NeptuneCallbackConfigurator
+#    args:
+#      callback_type: neptune.integrations.optuna:NeptuneCallback
+#      project: deepsense-ai/ragbits

examples/evaluation/document-search/advanced/config/pipeline/document_ingestion.yaml

@@ -0,0 +1,9 @@
+defaults:
+  - embedder: litellm
+  - providers: unstructured
+  - vector_store: chroma
+  - _self_
+
+type: ragbits.evaluate.pipelines.document_search:DocumentSearchWithIngestionPipeline
+ingest: true
+search: false

examples/evaluation/document-search/config/pipeline/document_search.yaml → examples/evaluation/document-search/advanced/config/pipeline/document_search.yaml

@@ -4,4 +4,7 @@ defaults:
   - vector_store: chroma
   - rephraser: noop
   - reranker: noop
-  - _self_
+  - _self_
+
+
+type: ragbits.evaluate.pipelines.document_search:DocumentSearchWithIngestionPipeline

examples/evaluation/document-search/config/pipeline/document_search_optimization.yaml → examples/evaluation/document-search/advanced/config/pipeline/document_search_optimization.yaml

@@ -8,3 +8,5 @@ defaults:
   - _self_
 
 type: ragbits.evaluate.pipelines.document_search:DocumentSearchWithIngestionPipeline
+ingest: true
+

examples/evaluation/document-search/evaluate.py → examples/evaluation/document-search/advanced/evaluate.py

@@ -8,14 +8,12 @@
 # ///
 import asyncio
 import logging
+from typing import cast
 
 import hydra
-from omegaconf import DictConfig
+from omegaconf import DictConfig, OmegaConf
 
 from ragbits.evaluate.evaluator import Evaluator
-from ragbits.evaluate.loaders import dataloader_factory
-from ragbits.evaluate.metrics import metric_set_factory
-from ragbits.evaluate.pipelines.document_search import DocumentSearchPipeline
 from ragbits.evaluate.utils import log_to_file, log_to_neptune, setup_neptune
 
 logging.getLogger("LiteLLM").setLevel(logging.ERROR)
@@ -31,20 +29,8 @@ async def bench(config: DictConfig) -> None:
         config: Hydra configuration.
     """
     run = setup_neptune(config)
-
-    log.info("Starting evaluation...")
-
-    dataloader = dataloader_factory(config.data)
-    pipeline = DocumentSearchPipeline(config.pipeline)
-    metrics = metric_set_factory(config.metrics)
-
-    evaluator = Evaluator()
-    results = await evaluator.compute(
-        pipeline=pipeline,
-        dataloader=dataloader,
-        metrics=metrics,
-    )
-
+    log.info("Starting the experiment...")
+    results = await Evaluator.run_experiment_from_config(config=cast(dict, OmegaConf.to_container(config)))
     output_dir = log_to_file(results)
     if run:
         log_to_neptune(run, results, output_dir)

examples/evaluation/document-search/advanced/optimize.py

@@ -0,0 +1,30 @@
+import sys
+from typing import cast
+
+import hydra
+from omegaconf import DictConfig, OmegaConf
+
+from ragbits.evaluate.optimizer import Optimizer
+from ragbits.evaluate.utils import log_optimization_to_file
+
+module = sys.modules[__name__]
+
+
+@hydra.main(config_path="config", config_name="optimization", version_base="3.2")
+def main(config: DictConfig) -> None:
+    """
+    Function running evaluation for all datasets and evaluation tasks defined in hydra config.
+
+    Args:
+        config: Hydra configuration.
+    """
+    exp_config = {
+        "optimizer": {"direction": "maximize", "n_trials": 10},
+        "experiment_config": cast(dict, OmegaConf.to_container(config)),
+    }
+    configs_with_scores = Optimizer.run_experiment_from_config(config=exp_config)
+    log_optimization_to_file(configs_with_scores)
+
+
+if __name__ == "__main__":
+    main()

examples/evaluation/document-search/basic/basic_evaluate.py

@@ -0,0 +1,82 @@
+# /// script
+# requires-python = ">=3.10"
+# dependencies = [
+#     "ragbits-document-search[huggingface]",
+#     "ragbits-core[chroma]",
+#     "hydra-core~=1.3.2",
+#     "unstructured[md]>=0.15.13",
+# ]
+# ///
+import asyncio
+import logging
+import uuid
+from pathlib import Path
+
+from ragbits.evaluate.evaluator import Evaluator
+from ragbits.evaluate.utils import log_to_file
+
+logging.getLogger("LiteLLM").setLevel(logging.ERROR)
+logging.getLogger("httpx").setLevel(logging.ERROR)
+log = logging.getLogger(__name__)
+
+
+async def evaluate() -> dict:
+    """
+    Basic example of document search evaluation.
+
+    """
+    log.info("Ingesting documents...")
+
+    config = {
+        "pipeline": {
+            "type": "ragbits.evaluate.pipelines.document_search:DocumentSearchWithIngestionPipeline",
+            "ingest": False,
+            "search": True,
+            "vector_store": {
+                "type": "ragbits.core.vector_stores.chroma:ChromaVectorStore",
+                "config": {
+                    "client": {"type": "PersistentClient", "config": {"path": "chroma"}},
+                    "index_name": "default",
+                    "distance_method": "l2",
+                    "default_options": {"k": 3, "max_distance": 1.2},
+                },
+            },
+            "providers": {
+                "txt": {"type": "ragbits.document_search.ingestion.providers.unstructured:UnstructuredDefaultProvider"}
+            },
+        },
+        "data": {
+            "type": "ragbits.evaluate.loaders.hf:HFDataLoader",
+            "options": {"name": "hf-docs-retrieval", "path": "micpst/hf-docs-retrieval", "split": "train"},
+        },
+        "metrics": [
+            {
+                "type": "ragbits.evaluate.metrics.document_search:DocumentSearchPrecisionRecallF1",
+                "matching_strategy": "RougeChunkMatch",
+                "options": {"threshold": 0.5},
+            }
+        ],
+        "neptune": {"project": "ragbits", "run": False},
+        "task": {"name": "default", "type": "document-search"},
+    }
+
+    results = await Evaluator.run_experiment_from_config(config=config)
+
+    log.info("Evaluation finished.")
+
+    return results
+
+
+def main() -> None:
+    """
+    Run the evaluation process.
+
+    """
+    results = asyncio.run(evaluate())
+    out_dir = Path(str(uuid.uuid4()))
+    out_dir.mkdir()
+    log_to_file(results, output_dir=out_dir)
+
+
+if __name__ == "__main__":
+    main()  # pylint: disable=no-value-for-parameter

examples/evaluation/document-search/basic/basic_ingest.py

@@ -0,0 +1,57 @@
+# /// script
+# requires-python = ">=3.10"
+# dependencies = [
+#     "ragbits-document-search[huggingface]",
+#     "ragbits-core[chroma]",
+#     "hydra-core~=1.3.2",
+#     "unstructured[md]>=0.15.13",
+# ]
+# ///
+import asyncio
+import logging
+
+from ragbits.evaluate.pipelines import pipeline_factory
+
+logging.getLogger("LiteLLM").setLevel(logging.ERROR)
+logging.getLogger("httpx").setLevel(logging.ERROR)
+log = logging.getLogger(__name__)
+
+
+async def ingest() -> None:
+    """
+    Ingest documents into the document search system.
+
+    Args:
+        config: Hydra configuration.
+    """
+    log.info("Ingesting documents...")
+
+    config = {
+        "type": "ragbits.evaluate.pipelines.document_search:DocumentSearchWithIngestionPipeline",
+        "ingest": True,
+        "search": False,
+        "answer_data_source": {"name": "hf-docs", "path": "micpst/hf-docs", "split": "train", "num_docs": 5},
+        "providers": {
+            "txt": {"type": "ragbits.document_search.ingestion.providers.unstructured:UnstructuredDefaultProvider"}
+        },
+    }
+
+    ingestor = pipeline_factory(config)  # type: ignore
+
+    await ingestor()
+
+    log.info("Ingestion finished.")
+
+
+def main() -> None:
+    """
+    Run the ingestion process.
+
+    Args:
+        config: Hydra configuration.
+    """
+    asyncio.run(ingest())
+
+
+if __name__ == "__main__":
+    main()  # pylint: disable=no-value-for-parameter

examples/evaluation/document-search/basic/basic_optimize.py

@@ -0,0 +1,59 @@
+import sys
+from pathlib import Path
+
+from ragbits.evaluate.optimizer import Optimizer
+from ragbits.evaluate.utils import log_optimization_to_file
+
+module = sys.modules[__name__]
+
+
+def main() -> None:
+    """
+    Function running evaluation for all datasets and evaluation tasks defined in config.
+    """
+    config = {
+        "pipeline": {
+            "type": "ragbits.evaluate.pipelines.document_search:DocumentSearchWithIngestionPipeline",
+            "ingest": True,
+            "search": True,
+            "answer_data_source": {
+                "name": "hf-docs",
+                "path": "micpst/hf-docs",
+                "split": "train",
+                "num_docs": 5,
+            },
+            "providers": {
+                "txt": {"type": "ragbits.document_search.ingestion.providers.unstructured:UnstructuredDefaultProvider"}
+            },
+            "embedder": {
+                "type": "ragbits.core.embeddings.litellm:LiteLLMEmbeddings",
+                "config": {
+                    "model": "text-embedding-3-small",
+                    "options": {
+                        "dimensions": {"optimize": True, "range": [32, 512]},
+                    },
+                },
+            },
+        },
+        "data": {
+            "type": "ragbits.evaluate.loaders.hf:HFDataLoader",
+            "options": {"name": "hf-docs-retrieval", "path": "micpst/hf-docs-retrieval", "split": "train"},
+        },
+        "metrics": [
+            {
+                "type": "ragbits.evaluate.metrics.document_search:DocumentSearchPrecisionRecallF1",
+                "matching_strategy": "RougeChunkMatch",
+                "options": {"threshold": 0.5},
+            }
+        ],
+    }
+    exp_config = {"optimizer": {"direction": "maximize", "n_trials": 10}, "experiment_config": config}
+    configs_with_scores = Optimizer.run_experiment_from_config(config=exp_config)
+
+    OUT_DIR = Path("basic_optimization")
+    OUT_DIR.mkdir()
+    log_optimization_to_file(configs_with_scores, output_dir=OUT_DIR)
+
+
+if __name__ == "__main__":
+    main()

packages/ragbits-core/src/ragbits/core/prompt/promptfoo.py

@@ -46,5 +46,5 @@ def generate_configs(
         target_path.mkdir()
     for prompt in prompts:
         with open(target_path / f"{prompt.__qualname__}.yaml", "w", encoding="utf-8") as f:
-            prompt_path = f'file://{prompt.__module__.replace(".", os.sep)}.py:{prompt.__qualname__}.to_promptfoo'
+            prompt_path = f"file://{prompt.__module__.replace('.', os.sep)}.py:{prompt.__qualname__}.to_promptfoo"
             yaml.dump({"prompts": [prompt_path]}, f)

packages/ragbits-core/src/ragbits/core/utils/config_handling.py

@@ -86,7 +86,6 @@ def subclass_from_config(cls, config: ObjectContructionConfig) -> Self:
         subclass = import_by_path(config.type, cls.default_module)
         if not issubclass(subclass, cls):
             raise InvalidConfigError(f"{subclass} is not a subclass of {cls}")
-
         return subclass.from_config(config.config)
 
     @classmethod

packages/ragbits-core/src/ragbits/core/utils/dict_transformations.py

@@ -98,7 +98,7 @@ def _decompose_key(key: str) -> tuple[str | int | None, str | int | None]:
                 _current_subkey = int(_key[start_subscript_index:end_subscript_index])
 
                 if len(_key[end_subscript_index:]) > 1:
-                    _current_subkey = f"{_current_subkey}.{_key[end_subscript_index + 2:]}"
+                    _current_subkey = f"{_current_subkey}.{_key[end_subscript_index + 2 :]}"
                 break
             elif char == ".":
                 split_work = _key.split(".", 1)