From 41b576fbc33f4bdd55d456abea08a07e2e6b30bd Mon Sep 17 00:00:00 2001 From: kdziedzic68 Date: Wed, 18 Dec 2024 10:45:20 +0100 Subject: [PATCH 01/13] run from config interface for evaluator --- .../config/pipeline/document_ingestion.yaml | 6 +++- .../config/pipeline/document_search.yaml | 5 +++- .../document_search_optimization.yaml | 2 ++ .../evaluation/document-search/evaluate.py | 16 ++--------- .../src/ragbits/evaluate/evaluator.py | 28 ++++++++++++++++--- .../ragbits/evaluate/pipelines/__init__.py | 15 ++++++++++ .../evaluate/pipelines/document_search.py | 9 +++--- .../src/ragbits/evaluate/utils.py | 2 +- 8 files changed, 58 insertions(+), 25 deletions(-) diff --git a/examples/evaluation/document-search/config/pipeline/document_ingestion.yaml b/examples/evaluation/document-search/config/pipeline/document_ingestion.yaml index edf49af0..32cd12a2 100644 --- a/examples/evaluation/document-search/config/pipeline/document_ingestion.yaml +++ b/examples/evaluation/document-search/config/pipeline/document_ingestion.yaml @@ -2,4 +2,8 @@ defaults: - embedder: litellm - providers: unstructured - vector_store: chroma - - _self_ \ No newline at end of file + - _self_ + +type: ragbits.evaluate.pipelines.document_search:DocumentSearchWithIngestionPipeline +ingest: true +search: false \ No newline at end of file diff --git a/examples/evaluation/document-search/config/pipeline/document_search.yaml b/examples/evaluation/document-search/config/pipeline/document_search.yaml index 8eaffd68..bc379e88 100644 --- a/examples/evaluation/document-search/config/pipeline/document_search.yaml +++ b/examples/evaluation/document-search/config/pipeline/document_search.yaml @@ -4,4 +4,7 @@ defaults: - vector_store: chroma - rephraser: noop - reranker: noop - - _self_ \ No newline at end of file + - _self_ + + +type: ragbits.evaluate.pipelines.document_search:DocumentSearchWithIngestionPipeline diff --git a/examples/evaluation/document-search/config/pipeline/document_search_optimization.yaml b/examples/evaluation/document-search/config/pipeline/document_search_optimization.yaml index 8364c038..34bd0542 100644 --- a/examples/evaluation/document-search/config/pipeline/document_search_optimization.yaml +++ b/examples/evaluation/document-search/config/pipeline/document_search_optimization.yaml @@ -8,3 +8,5 @@ defaults: - _self_ type: ragbits.evaluate.pipelines.document_search:DocumentSearchWithIngestionPipeline +ingest: true + diff --git a/examples/evaluation/document-search/evaluate.py b/examples/evaluation/document-search/evaluate.py index d421ce59..0fec74c7 100644 --- a/examples/evaluation/document-search/evaluate.py +++ b/examples/evaluation/document-search/evaluate.py @@ -31,20 +31,8 @@ async def bench(config: DictConfig) -> None: config: Hydra configuration. """ run = setup_neptune(config) - - log.info("Starting evaluation...") - - dataloader = dataloader_factory(config.data) - pipeline = DocumentSearchPipeline(config.pipeline) - metrics = metric_set_factory(config.metrics) - - evaluator = Evaluator() - results = await evaluator.compute( - pipeline=pipeline, - dataloader=dataloader, - metrics=metrics, - ) - + log.info("Starting the experiment...") + results = await Evaluator.run_experiment_from_config(config=config) output_dir = log_to_file(results) if run: log_to_neptune(run, results, output_dir) diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/evaluator.py b/packages/ragbits-evaluate/src/ragbits/evaluate/evaluator.py index ea1d078a..446dbe10 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/evaluator.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/evaluator.py @@ -3,10 +3,14 @@ from dataclasses import asdict from typing import Any +from omegaconf import DictConfig from tqdm.asyncio import tqdm -from ragbits.evaluate.loaders.base import DataLoader -from ragbits.evaluate.metrics.base import MetricSet +from .loaders import dataloader_factory +from .metrics import metric_set_factory +from .loaders.base import DataLoader +from .metrics.base import MetricSet +from .pipelines import pipeline_factory from ragbits.evaluate.pipelines.base import EvaluationPipeline, EvaluationResult @@ -19,7 +23,7 @@ async def compute( self, pipeline: EvaluationPipeline, dataloader: DataLoader, - metrics: MetricSet, + metrics: MetricSet | None, ) -> dict[str, Any]: """ Compute the evaluation results for the given pipeline and data. @@ -34,7 +38,7 @@ async def compute( """ dataset = await dataloader.load() results, perf_results = await self._call_pipeline(pipeline, dataset) - computed_metrics = self._compute_metrics(metrics, results) + computed_metrics = self._compute_metrics(metrics, results) if metrics else {} processed_results = self._results_processor(results) return { @@ -43,6 +47,22 @@ async def compute( **processed_results, } + @classmethod + async def run_experiment_from_config(cls, config: DictConfig) -> dict[str, Any] | None: + dataloader = dataloader_factory(config.data) + pipeline = pipeline_factory(config.pipeline) + + metric_config = config.get("metrics", None) + metrics = metric_set_factory(metric_config) if metric_config is not None else None + + evaluator = cls() + results = await evaluator.compute( + pipeline=pipeline, + dataloader=dataloader, + metrics=metrics, + ) + return results + async def _call_pipeline( self, pipeline: EvaluationPipeline, diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/__init__.py b/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/__init__.py index e69de29b..cdfb8e02 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/__init__.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/__init__.py @@ -0,0 +1,15 @@ +import sys + +from omegaconf import DictConfig + +from ragbits.core.utils.config_handling import import_by_path + +from .base import EvaluationPipeline + +module = sys.modules[__name__] + + +def pipeline_factory(pipeline_config: DictConfig) -> EvaluationPipeline: + pipeline_module = import_by_path(pipeline_config.type, module) + pipeline = pipeline_module(pipeline_config) + return pipeline diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/document_search.py b/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/document_search.py index 6b15b46f..8e76aaf1 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/document_search.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/document_search.py @@ -38,7 +38,7 @@ def document_search(self) -> "DocumentSearch": """ return DocumentSearch.from_config(self.config) # type: ignore - async def __call__(self, data: dict) -> DocumentSearchResult: + async def __call__(self, data: dict) -> DocumentSearchResult | None: """ Runs the document search evaluation pipeline. @@ -68,7 +68,7 @@ def __init__(self, config: DictConfig | None = None) -> None: self._ingested = False self._lock = asyncio.Lock() - async def __call__(self, data: dict) -> DocumentSearchResult: + async def __call__(self, data: dict) -> DocumentSearchResult | None: """ Queries a vector store with given data Ingests the corpus to the store if has not been done @@ -78,10 +78,11 @@ async def __call__(self, data: dict) -> DocumentSearchResult: DocumentSearchResult - query result """ async with self._lock: - if not self._ingested: + if not self._ingested and self.config.get("ingest", False): await self._ingest_documents() self._ingested = True - return await super().__call__(data) + if self.config.get("search", True): + return await super().__call__(data) async def _ingest_documents(self) -> None: documents = await tqdm.gather( diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/utils.py b/packages/ragbits-evaluate/src/ragbits/evaluate/utils.py index f2ea8fb8..b24883a8 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/utils.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/utils.py @@ -101,7 +101,7 @@ def setup_neptune(config: DictConfig) -> Run | None: Returns: The Neptune run. """ - if config.neptune.run: + if config.get("neptune", {}).get("run"): run = Run( project=config.neptune.project, tags=[ From 83ed7e7bfc50aef18357b3bf376c0d56edc0eddc Mon Sep 17 00:00:00 2001 From: kdziedzic68 Date: Wed, 18 Dec 2024 13:56:04 +0100 Subject: [PATCH 02/13] optimize basic --- .../document-search/{ => advanced}/README.md | 0 .../{ => advanced}/config/data/corpus.yaml | 0 .../{ => advanced}/config/data/qa.yaml | 0 .../config/embedder/litellm.yaml | 0 .../config/experiments/chunking-1000.yaml | 0 .../config/experiments/chunking-250.yaml | 0 .../config/experiments/chunking-500.yaml | 0 .../{ => advanced}/config/ingestion.yaml | 0 .../{ => advanced}/config/optimization.yaml | 10 +-- .../pipeline/answer_data_source/corpus.yaml | 0 .../config/pipeline/document_ingestion.yaml | 0 .../config/pipeline/document_search.yaml | 0 .../document_search_optimization.yaml | 0 .../config/pipeline/embedder/litellm.yaml | 0 .../embedder/litellm_opt_template.yaml | 0 .../pipeline/providers/unstructured.yaml | 0 .../providers/unstructured_opt_template.yaml | 0 .../config/pipeline/rephraser/noop.yaml | 0 .../config/pipeline/reranker/noop.yaml | 0 .../config/pipeline/vector_store/chroma.yaml | 0 .../config/providers/unstructured.yaml | 0 .../{ => advanced}/config/rephraser/noop.yaml | 0 .../{ => advanced}/config/reranker/noop.yaml | 0 .../{ => advanced}/config/retrieval.yaml | 0 .../config/vector_store/chroma.yaml | 0 .../{ => advanced}/evaluate.py | 3 - .../document-search/{ => advanced}/ingest.py | 0 .../document-search/advanced/optimize.py | 26 +++++++ .../document-search/basic/evaluate.py | 78 +++++++++++++++++++ .../document-search/basic/ingest.py | 60 ++++++++++++++ .../document-search/basic/optimize.py | 51 ++++++++++++ .../evaluation/document-search/optimize.py | 45 ----------- .../src/ragbits/core/utils/config_handling.py | 1 - .../ragbits/core/vector_stores/__init__.py | 12 ++- .../src/ragbits/document_search/_main.py | 7 +- .../src/ragbits/evaluate/evaluator.py | 9 ++- .../src/ragbits/evaluate/optimizer.py | 39 +++++++++- .../evaluate/pipelines/document_search.py | 6 +- 38 files changed, 285 insertions(+), 62 deletions(-) rename examples/evaluation/document-search/{ => advanced}/README.md (100%) rename examples/evaluation/document-search/{ => advanced}/config/data/corpus.yaml (100%) rename examples/evaluation/document-search/{ => advanced}/config/data/qa.yaml (100%) rename examples/evaluation/document-search/{ => advanced}/config/embedder/litellm.yaml (100%) rename examples/evaluation/document-search/{ => advanced}/config/experiments/chunking-1000.yaml (100%) rename examples/evaluation/document-search/{ => advanced}/config/experiments/chunking-250.yaml (100%) rename examples/evaluation/document-search/{ => advanced}/config/experiments/chunking-500.yaml (100%) rename examples/evaluation/document-search/{ => advanced}/config/ingestion.yaml (100%) rename examples/evaluation/document-search/{ => advanced}/config/optimization.yaml (70%) rename examples/evaluation/document-search/{ => advanced}/config/pipeline/answer_data_source/corpus.yaml (100%) rename examples/evaluation/document-search/{ => advanced}/config/pipeline/document_ingestion.yaml (100%) rename examples/evaluation/document-search/{ => advanced}/config/pipeline/document_search.yaml (100%) rename examples/evaluation/document-search/{ => advanced}/config/pipeline/document_search_optimization.yaml (100%) rename examples/evaluation/document-search/{ => advanced}/config/pipeline/embedder/litellm.yaml (100%) rename examples/evaluation/document-search/{ => advanced}/config/pipeline/embedder/litellm_opt_template.yaml (100%) rename examples/evaluation/document-search/{ => advanced}/config/pipeline/providers/unstructured.yaml (100%) rename examples/evaluation/document-search/{ => advanced}/config/pipeline/providers/unstructured_opt_template.yaml (100%) rename examples/evaluation/document-search/{ => advanced}/config/pipeline/rephraser/noop.yaml (100%) rename examples/evaluation/document-search/{ => advanced}/config/pipeline/reranker/noop.yaml (100%) rename examples/evaluation/document-search/{ => advanced}/config/pipeline/vector_store/chroma.yaml (100%) rename examples/evaluation/document-search/{ => advanced}/config/providers/unstructured.yaml (100%) rename examples/evaluation/document-search/{ => advanced}/config/rephraser/noop.yaml (100%) rename examples/evaluation/document-search/{ => advanced}/config/reranker/noop.yaml (100%) rename examples/evaluation/document-search/{ => advanced}/config/retrieval.yaml (100%) rename examples/evaluation/document-search/{ => advanced}/config/vector_store/chroma.yaml (100%) rename examples/evaluation/document-search/{ => advanced}/evaluate.py (88%) rename examples/evaluation/document-search/{ => advanced}/ingest.py (100%) create mode 100644 examples/evaluation/document-search/advanced/optimize.py create mode 100644 examples/evaluation/document-search/basic/evaluate.py create mode 100644 examples/evaluation/document-search/basic/ingest.py create mode 100644 examples/evaluation/document-search/basic/optimize.py delete mode 100644 examples/evaluation/document-search/optimize.py diff --git a/examples/evaluation/document-search/README.md b/examples/evaluation/document-search/advanced/README.md similarity index 100% rename from examples/evaluation/document-search/README.md rename to examples/evaluation/document-search/advanced/README.md diff --git a/examples/evaluation/document-search/config/data/corpus.yaml b/examples/evaluation/document-search/advanced/config/data/corpus.yaml similarity index 100% rename from examples/evaluation/document-search/config/data/corpus.yaml rename to examples/evaluation/document-search/advanced/config/data/corpus.yaml diff --git a/examples/evaluation/document-search/config/data/qa.yaml b/examples/evaluation/document-search/advanced/config/data/qa.yaml similarity index 100% rename from examples/evaluation/document-search/config/data/qa.yaml rename to examples/evaluation/document-search/advanced/config/data/qa.yaml diff --git a/examples/evaluation/document-search/config/embedder/litellm.yaml b/examples/evaluation/document-search/advanced/config/embedder/litellm.yaml similarity index 100% rename from examples/evaluation/document-search/config/embedder/litellm.yaml rename to examples/evaluation/document-search/advanced/config/embedder/litellm.yaml diff --git a/examples/evaluation/document-search/config/experiments/chunking-1000.yaml b/examples/evaluation/document-search/advanced/config/experiments/chunking-1000.yaml similarity index 100% rename from examples/evaluation/document-search/config/experiments/chunking-1000.yaml rename to examples/evaluation/document-search/advanced/config/experiments/chunking-1000.yaml diff --git a/examples/evaluation/document-search/config/experiments/chunking-250.yaml b/examples/evaluation/document-search/advanced/config/experiments/chunking-250.yaml similarity index 100% rename from examples/evaluation/document-search/config/experiments/chunking-250.yaml rename to examples/evaluation/document-search/advanced/config/experiments/chunking-250.yaml diff --git a/examples/evaluation/document-search/config/experiments/chunking-500.yaml b/examples/evaluation/document-search/advanced/config/experiments/chunking-500.yaml similarity index 100% rename from examples/evaluation/document-search/config/experiments/chunking-500.yaml rename to examples/evaluation/document-search/advanced/config/experiments/chunking-500.yaml diff --git a/examples/evaluation/document-search/config/ingestion.yaml b/examples/evaluation/document-search/advanced/config/ingestion.yaml similarity index 100% rename from examples/evaluation/document-search/config/ingestion.yaml rename to examples/evaluation/document-search/advanced/config/ingestion.yaml diff --git a/examples/evaluation/document-search/config/optimization.yaml b/examples/evaluation/document-search/advanced/config/optimization.yaml similarity index 70% rename from examples/evaluation/document-search/config/optimization.yaml rename to examples/evaluation/document-search/advanced/config/optimization.yaml index 53ff3424..807676cc 100644 --- a/examples/evaluation/document-search/config/optimization.yaml +++ b/examples/evaluation/document-search/advanced/config/optimization.yaml @@ -19,8 +19,8 @@ metrics: threshold: 0.5 -callbacks: - - type: ragbits.evaluate.callbacks.neptune:NeptuneCallbackConfigurator - args: - callback_type: neptune.integrations.optuna:NeptuneCallback - project: deepsense-ai/ragbits +#callbacks: +# - type: ragbits.evaluate.callbacks.neptune:NeptuneCallbackConfigurator +# args: +# callback_type: neptune.integrations.optuna:NeptuneCallback +# project: deepsense-ai/ragbits diff --git a/examples/evaluation/document-search/config/pipeline/answer_data_source/corpus.yaml b/examples/evaluation/document-search/advanced/config/pipeline/answer_data_source/corpus.yaml similarity index 100% rename from examples/evaluation/document-search/config/pipeline/answer_data_source/corpus.yaml rename to examples/evaluation/document-search/advanced/config/pipeline/answer_data_source/corpus.yaml diff --git a/examples/evaluation/document-search/config/pipeline/document_ingestion.yaml b/examples/evaluation/document-search/advanced/config/pipeline/document_ingestion.yaml similarity index 100% rename from examples/evaluation/document-search/config/pipeline/document_ingestion.yaml rename to examples/evaluation/document-search/advanced/config/pipeline/document_ingestion.yaml diff --git a/examples/evaluation/document-search/config/pipeline/document_search.yaml b/examples/evaluation/document-search/advanced/config/pipeline/document_search.yaml similarity index 100% rename from examples/evaluation/document-search/config/pipeline/document_search.yaml rename to examples/evaluation/document-search/advanced/config/pipeline/document_search.yaml diff --git a/examples/evaluation/document-search/config/pipeline/document_search_optimization.yaml b/examples/evaluation/document-search/advanced/config/pipeline/document_search_optimization.yaml similarity index 100% rename from examples/evaluation/document-search/config/pipeline/document_search_optimization.yaml rename to examples/evaluation/document-search/advanced/config/pipeline/document_search_optimization.yaml diff --git a/examples/evaluation/document-search/config/pipeline/embedder/litellm.yaml b/examples/evaluation/document-search/advanced/config/pipeline/embedder/litellm.yaml similarity index 100% rename from examples/evaluation/document-search/config/pipeline/embedder/litellm.yaml rename to examples/evaluation/document-search/advanced/config/pipeline/embedder/litellm.yaml diff --git a/examples/evaluation/document-search/config/pipeline/embedder/litellm_opt_template.yaml b/examples/evaluation/document-search/advanced/config/pipeline/embedder/litellm_opt_template.yaml similarity index 100% rename from examples/evaluation/document-search/config/pipeline/embedder/litellm_opt_template.yaml rename to examples/evaluation/document-search/advanced/config/pipeline/embedder/litellm_opt_template.yaml diff --git a/examples/evaluation/document-search/config/pipeline/providers/unstructured.yaml b/examples/evaluation/document-search/advanced/config/pipeline/providers/unstructured.yaml similarity index 100% rename from examples/evaluation/document-search/config/pipeline/providers/unstructured.yaml rename to examples/evaluation/document-search/advanced/config/pipeline/providers/unstructured.yaml diff --git a/examples/evaluation/document-search/config/pipeline/providers/unstructured_opt_template.yaml b/examples/evaluation/document-search/advanced/config/pipeline/providers/unstructured_opt_template.yaml similarity index 100% rename from examples/evaluation/document-search/config/pipeline/providers/unstructured_opt_template.yaml rename to examples/evaluation/document-search/advanced/config/pipeline/providers/unstructured_opt_template.yaml diff --git a/examples/evaluation/document-search/config/pipeline/rephraser/noop.yaml b/examples/evaluation/document-search/advanced/config/pipeline/rephraser/noop.yaml similarity index 100% rename from examples/evaluation/document-search/config/pipeline/rephraser/noop.yaml rename to examples/evaluation/document-search/advanced/config/pipeline/rephraser/noop.yaml diff --git a/examples/evaluation/document-search/config/pipeline/reranker/noop.yaml b/examples/evaluation/document-search/advanced/config/pipeline/reranker/noop.yaml similarity index 100% rename from examples/evaluation/document-search/config/pipeline/reranker/noop.yaml rename to examples/evaluation/document-search/advanced/config/pipeline/reranker/noop.yaml diff --git a/examples/evaluation/document-search/config/pipeline/vector_store/chroma.yaml b/examples/evaluation/document-search/advanced/config/pipeline/vector_store/chroma.yaml similarity index 100% rename from examples/evaluation/document-search/config/pipeline/vector_store/chroma.yaml rename to examples/evaluation/document-search/advanced/config/pipeline/vector_store/chroma.yaml diff --git a/examples/evaluation/document-search/config/providers/unstructured.yaml b/examples/evaluation/document-search/advanced/config/providers/unstructured.yaml similarity index 100% rename from examples/evaluation/document-search/config/providers/unstructured.yaml rename to examples/evaluation/document-search/advanced/config/providers/unstructured.yaml diff --git a/examples/evaluation/document-search/config/rephraser/noop.yaml b/examples/evaluation/document-search/advanced/config/rephraser/noop.yaml similarity index 100% rename from examples/evaluation/document-search/config/rephraser/noop.yaml rename to examples/evaluation/document-search/advanced/config/rephraser/noop.yaml diff --git a/examples/evaluation/document-search/config/reranker/noop.yaml b/examples/evaluation/document-search/advanced/config/reranker/noop.yaml similarity index 100% rename from examples/evaluation/document-search/config/reranker/noop.yaml rename to examples/evaluation/document-search/advanced/config/reranker/noop.yaml diff --git a/examples/evaluation/document-search/config/retrieval.yaml b/examples/evaluation/document-search/advanced/config/retrieval.yaml similarity index 100% rename from examples/evaluation/document-search/config/retrieval.yaml rename to examples/evaluation/document-search/advanced/config/retrieval.yaml diff --git a/examples/evaluation/document-search/config/vector_store/chroma.yaml b/examples/evaluation/document-search/advanced/config/vector_store/chroma.yaml similarity index 100% rename from examples/evaluation/document-search/config/vector_store/chroma.yaml rename to examples/evaluation/document-search/advanced/config/vector_store/chroma.yaml diff --git a/examples/evaluation/document-search/evaluate.py b/examples/evaluation/document-search/advanced/evaluate.py similarity index 88% rename from examples/evaluation/document-search/evaluate.py rename to examples/evaluation/document-search/advanced/evaluate.py index 0fec74c7..456e08b0 100644 --- a/examples/evaluation/document-search/evaluate.py +++ b/examples/evaluation/document-search/advanced/evaluate.py @@ -13,9 +13,6 @@ from omegaconf import DictConfig from ragbits.evaluate.evaluator import Evaluator -from ragbits.evaluate.loaders import dataloader_factory -from ragbits.evaluate.metrics import metric_set_factory -from ragbits.evaluate.pipelines.document_search import DocumentSearchPipeline from ragbits.evaluate.utils import log_to_file, log_to_neptune, setup_neptune logging.getLogger("LiteLLM").setLevel(logging.ERROR) diff --git a/examples/evaluation/document-search/ingest.py b/examples/evaluation/document-search/advanced/ingest.py similarity index 100% rename from examples/evaluation/document-search/ingest.py rename to examples/evaluation/document-search/advanced/ingest.py diff --git a/examples/evaluation/document-search/advanced/optimize.py b/examples/evaluation/document-search/advanced/optimize.py new file mode 100644 index 00000000..4644f331 --- /dev/null +++ b/examples/evaluation/document-search/advanced/optimize.py @@ -0,0 +1,26 @@ +import sys + +import hydra +from omegaconf import DictConfig, OmegaConf + +from ragbits.evaluate.optimizer import Optimizer +from ragbits.evaluate.utils import log_optimization_to_file + +module = sys.modules[__name__] + + +@hydra.main(config_path="config", config_name="optimization", version_base="3.2") +def main(config: DictConfig) -> None: + """ + Function running evaluation for all datasets and evaluation tasks defined in hydra config. + + Args: + config: Hydra configuration. + """ + config = {"optimizer": OmegaConf.create({"direction": "maximize", "n_trials": 10}), "experiment_config": config} + configs_with_scores = Optimizer.run_experiment_from_config(config=config) + log_optimization_to_file(configs_with_scores) + + +if __name__ == "__main__": + main() diff --git a/examples/evaluation/document-search/basic/evaluate.py b/examples/evaluation/document-search/basic/evaluate.py new file mode 100644 index 00000000..151da795 --- /dev/null +++ b/examples/evaluation/document-search/basic/evaluate.py @@ -0,0 +1,78 @@ +# /// script +# requires-python = ">=3.10" +# dependencies = [ +# "ragbits-document-search[huggingface]", +# "ragbits-core[chroma]", +# "hydra-core~=1.3.2", +# "unstructured[md]>=0.15.13", +# ] +# /// +import asyncio +import logging +import uuid +from pathlib import Path + +from omegaconf import OmegaConf +from ragbits.evaluate.utils import log_to_file +from ragbits.evaluate.evaluator import Evaluator + +logging.getLogger("LiteLLM").setLevel(logging.ERROR) +logging.getLogger("httpx").setLevel(logging.ERROR) +log = logging.getLogger(__name__) + + +async def evaluate() -> dict: + """ + Basic example of document search evaluation. + + """ + log.info("Ingesting documents...") + + config = OmegaConf.create( + { + "pipeline": { + "type": "ragbits.evaluate.pipelines.document_search:DocumentSearchWithIngestionPipeline", + "ingest": False, + "search": True, + "providers": { + "txt": { + "type": "ragbits.document_search.ingestion.providers.unstructured:UnstructuredDefaultProvider" + } + }, + }, + "data": { + "type": "ragbits.evaluate.loaders.hf:HFDataLoader", + "options": {"name": "hf-docs-retrieval", "path": "micpst/hf-docs-retrieval", "split": "train"}, + }, + "metrics": [ + { + "type": "ragbits.evaluate.metrics.document_search:DocumentSearchPrecisionRecallF1", + "matching_strategy": "RougeChunkMatch", + "options": {"threshold": 0.5}, + } + ], + "neptune": {"project": "ragbits", "run": False}, + "task": {"name": "default", "type": "document-search"}, + } + ) + + results = await Evaluator.run_experiment_from_config(config=config) + + log.info("Evaluation finished.") + + return results + + +def main() -> None: + """ + Run the evaluation process. + + """ + results = asyncio.run(evaluate()) + out_dir = Path(str(uuid.uuid4())) + out_dir.mkdir() + log_to_file(results, output_dir=out_dir) + + +if __name__ == "__main__": + main() # pylint: disable=no-value-for-parameter diff --git a/examples/evaluation/document-search/basic/ingest.py b/examples/evaluation/document-search/basic/ingest.py new file mode 100644 index 00000000..77dfff5c --- /dev/null +++ b/examples/evaluation/document-search/basic/ingest.py @@ -0,0 +1,60 @@ +# /// script +# requires-python = ">=3.10" +# dependencies = [ +# "ragbits-document-search[huggingface]", +# "ragbits-core[chroma]", +# "hydra-core~=1.3.2", +# "unstructured[md]>=0.15.13", +# ] +# /// +import asyncio +import logging + +from omegaconf import OmegaConf +from ragbits.evaluate.pipelines import pipeline_factory + +logging.getLogger("LiteLLM").setLevel(logging.ERROR) +logging.getLogger("httpx").setLevel(logging.ERROR) +log = logging.getLogger(__name__) + + +async def ingest() -> None: + """ + Ingest documents into the document search system. + + Args: + config: Hydra configuration. + """ + log.info("Ingesting documents...") + + config = OmegaConf.create( + { + "type": "ragbits.evaluate.pipelines.document_search:DocumentSearchWithIngestionPipeline", + "ingest": True, + "search": False, + "answer_data_source": {"name": "hf-docs", "path": "micpst/hf-docs", "split": "train", "num_docs": 5}, + "providers": { + "txt": {"type": "ragbits.document_search.ingestion.providers.unstructured:UnstructuredDefaultProvider"} + }, + } + ) + + ingestor = pipeline_factory(config) # type: ignore + + await ingestor() + + log.info("Ingestion finished.") + + +def main() -> None: + """ + Run the ingestion process. + + Args: + config: Hydra configuration. + """ + asyncio.run(ingest()) + + +if __name__ == "__main__": + main() # pylint: disable=no-value-for-parameter diff --git a/examples/evaluation/document-search/basic/optimize.py b/examples/evaluation/document-search/basic/optimize.py new file mode 100644 index 00000000..d835d090 --- /dev/null +++ b/examples/evaluation/document-search/basic/optimize.py @@ -0,0 +1,51 @@ +import sys + +from omegaconf import DictConfig, OmegaConf + +from ragbits.evaluate.optimizer import Optimizer +from ragbits.evaluate.utils import log_optimization_to_file + +module = sys.modules[__name__] + + +def main() -> None: + """ + Function running evaluation for all datasets and evaluation tasks defined in config. + """ + + config = OmegaConf.create( + { + "pipeline": { + "type": "ragbits.evaluate.pipelines.document_search:DocumentSearchWithIngestionPipeline", + "ingest": True, + "search": True, + "embedder": { + "model": "text-embedding-3-small", + "options": {"dimensions": {"optimize": True, "range": [32, 512]}, "encoding_format": float}, + "providers": { + "txt": { + "type": "ragbits.document_search.ingestion.providers.unstructured:UnstructuredDefaultProvider" + } + }, + }, + }, + "data": { + "type": "ragbits.evaluate.loaders.hf:HFDataLoader", + "options": {"name": "hf-docs-retrieval", "path": "micpst/hf-docs-retrieval", "split": "train"}, + }, + "metrics": [ + { + "type": "ragbits.evaluate.metrics.document_search:DocumentSearchPrecisionRecallF1", + "matching_strategy": "RougeChunkMatch", + "options": {"threshold": 0.5}, + } + ], + } + ) + config = {"optimizer": OmegaConf.create({"direction": "maximize", "n_trials": 10}), "experiment_config": config} + configs_with_scores = Optimizer.run_experiment_from_config(config=config) + log_optimization_to_file(configs_with_scores) + + +if __name__ == "__main__": + main() diff --git a/examples/evaluation/document-search/optimize.py b/examples/evaluation/document-search/optimize.py deleted file mode 100644 index 521e4e41..00000000 --- a/examples/evaluation/document-search/optimize.py +++ /dev/null @@ -1,45 +0,0 @@ -import sys - -import hydra -from omegaconf import DictConfig, OmegaConf - -from ragbits.core.utils.config_handling import import_by_path -from ragbits.evaluate.loaders import dataloader_factory -from ragbits.evaluate.metrics import metric_set_factory -from ragbits.evaluate.optimizer import Optimizer -from ragbits.evaluate.utils import log_optimization_to_file - -module = sys.modules[__name__] - - -@hydra.main(config_path="config", config_name="optimization", version_base="3.2") -def main(config: DictConfig) -> None: - """ - Function running evaluation for all datasets and evaluation tasks defined in hydra config. - - Args: - config: Hydra configuration. - """ - dataloader = dataloader_factory(config.data) - pipeline_class = import_by_path(config.pipeline.type, module) - metrics = metric_set_factory(config.metrics) - callback_configurators = None - if getattr(config, "callbacks", None): - callback_configurators = [ - import_by_path(callback_cfg.type, module)(callback_cfg.args) for callback_cfg in config.callbacks - ] - - optimization_cfg = OmegaConf.create({"direction": "maximize", "n_trials": 10}) - optimizer = Optimizer(cfg=optimization_cfg) - configs_with_scores = optimizer.optimize( - pipeline_class=pipeline_class, - config_with_params=config.pipeline, - metrics=metrics, - dataloader=dataloader, - callback_configurators=callback_configurators, - ) - log_optimization_to_file(configs_with_scores) - - -if __name__ == "__main__": - main() diff --git a/packages/ragbits-core/src/ragbits/core/utils/config_handling.py b/packages/ragbits-core/src/ragbits/core/utils/config_handling.py index 861ffff4..7226b7b4 100644 --- a/packages/ragbits-core/src/ragbits/core/utils/config_handling.py +++ b/packages/ragbits-core/src/ragbits/core/utils/config_handling.py @@ -86,7 +86,6 @@ def subclass_from_config(cls, config: ObjectContructionConfig) -> Self: subclass = import_by_path(config.type, cls.default_module) if not issubclass(subclass, cls): raise InvalidConfigError(f"{subclass} is not a subclass of {cls}") - return subclass.from_config(config.config) @classmethod diff --git a/packages/ragbits-core/src/ragbits/core/vector_stores/__init__.py b/packages/ragbits-core/src/ragbits/core/vector_stores/__init__.py index 7a85def1..e27b6690 100644 --- a/packages/ragbits-core/src/ragbits/core/vector_stores/__init__.py +++ b/packages/ragbits-core/src/ragbits/core/vector_stores/__init__.py @@ -1,4 +1,14 @@ from ragbits.core.vector_stores.base import VectorStore, VectorStoreEntry, VectorStoreOptions, WhereQuery from ragbits.core.vector_stores.in_memory import InMemoryVectorStore +from ragbits.core.vector_stores.chroma import ChromaVectorStore +from ragbits.core.vector_stores.qdrant import QdrantVectorStore -__all__ = ["InMemoryVectorStore", "VectorStore", "VectorStoreEntry", "VectorStoreOptions", "WhereQuery"] +__all__ = [ + "InMemoryVectorStore", + "VectorStore", + "VectorStoreEntry", + "VectorStoreOptions", + "WhereQuery", + "ChromaVectorStore", + "QdrantVectorStore", +] diff --git a/packages/ragbits-document-search/src/ragbits/document_search/_main.py b/packages/ragbits-document-search/src/ragbits/document_search/_main.py index 5db3f289..afe2eaa5 100644 --- a/packages/ragbits-document-search/src/ragbits/document_search/_main.py +++ b/packages/ragbits-document-search/src/ragbits/document_search/_main.py @@ -39,8 +39,11 @@ class DocumentSearchConfig(BaseModel): Schema for for the dict taken by DocumentSearch.from_config method. """ - embedder: ObjectContructionConfig - vector_store: ObjectContructionConfig + embedder: ObjectContructionConfig = ObjectContructionConfig(type="LiteLLMEmbeddings") + vector_store: ObjectContructionConfig = ObjectContructionConfig( + type="ChromaVectorStore", + config={"client": {"type": "PersistentClient"}, "index_name": "default"}, + ) rephraser: ObjectContructionConfig = ObjectContructionConfig(type="NoopQueryRephraser") reranker: ObjectContructionConfig = ObjectContructionConfig(type="NoopReranker") processing_strategy: ObjectContructionConfig = ObjectContructionConfig(type="SequentialProcessing") diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/evaluator.py b/packages/ragbits-evaluate/src/ragbits/evaluate/evaluator.py index 446dbe10..4f7e5db0 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/evaluator.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/evaluator.py @@ -48,7 +48,14 @@ async def compute( } @classmethod - async def run_experiment_from_config(cls, config: DictConfig) -> dict[str, Any] | None: + async def run_experiment_from_config(cls, config: DictConfig) -> dict[str, Any]: + """ + Runs the evaluation experiment basing on configuration + Args: + config: DictConfig - soe config + Returns: + dictionary of metrics with scores + """ dataloader = dataloader_factory(config.data) pipeline = pipeline_factory(config.pipeline) diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/optimizer.py b/packages/ragbits-evaluate/src/ragbits/evaluate/optimizer.py index 80a552b1..f730f8c8 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/optimizer.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/optimizer.py @@ -1,11 +1,15 @@ import asyncio +import sys import warnings from copy import deepcopy from typing import Any import optuna -from omegaconf import DictConfig, ListConfig +from omegaconf import DictConfig, ListConfig, OmegaConf +from ragbits.core.utils.config_handling import import_by_path +from .loaders import dataloader_factory +from .metrics import metric_set_factory from .callbacks.base import CallbackConfigurator from .evaluator import Evaluator from .loaders.base import DataLoader @@ -13,6 +17,9 @@ from .pipelines.base import EvaluationPipeline +module = sys.modules[__name__] + + class Optimizer: """ Class for optimization @@ -26,6 +33,36 @@ def __init__(self, cfg: DictConfig): # TODO check how optuna handles parallelism. discuss if we want to have parallel studies self._choices_cache: dict[str, list[Any]] = {} + @classmethod + def run_experiment_from_config(cls, config: dict[str, DictConfig]) -> list[tuple[DictConfig, float, dict[str, float]]]: + """ + Runs the optimization experiment configured with config object + Args: + config: dict + Returns: + list of configs with scores + """ + optimizer_config = config["optimizer"] + config = config["experiment_config"] + dataloader = dataloader_factory(config.data) + pipeline_class = import_by_path(config.pipeline.type, module) + metrics = metric_set_factory(config.metrics) + callback_configurators = None + if getattr(config, "callbacks", None): + callback_configurators = [ + import_by_path(callback_cfg.type, module)(callback_cfg.args) for callback_cfg in config.callbacks + ] + + optimizer = cls(cfg=optimizer_config) + configs_with_scores = optimizer.optimize( + pipeline_class=pipeline_class, + config_with_params=config.pipeline, + metrics=metrics, + dataloader=dataloader, + callback_configurators=callback_configurators, + ) + return configs_with_scores + def optimize( self, pipeline_class: type[EvaluationPipeline], diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/document_search.py b/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/document_search.py index 8e76aaf1..6ce1c89a 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/document_search.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/document_search.py @@ -38,7 +38,7 @@ def document_search(self) -> "DocumentSearch": """ return DocumentSearch.from_config(self.config) # type: ignore - async def __call__(self, data: dict) -> DocumentSearchResult | None: + async def __call__(self, data: dict | None) -> DocumentSearchResult | None: """ Runs the document search evaluation pipeline. @@ -64,11 +64,11 @@ class DocumentSearchWithIngestionPipeline(DocumentSearchPipeline): def __init__(self, config: DictConfig | None = None) -> None: super().__init__(config) - self.config.vector_store.config.index_name = str(uuid.uuid4()) + # self.config.vector_store.config.index_name = str(uuid.uuid4()) self._ingested = False self._lock = asyncio.Lock() - async def __call__(self, data: dict) -> DocumentSearchResult | None: + async def __call__(self, data: dict | None = None) -> DocumentSearchResult | None: """ Queries a vector store with given data Ingests the corpus to the store if has not been done From 070548adff8d0b285e417cdec0ce904b459945b0 Mon Sep 17 00:00:00 2001 From: kdziedzic68 Date: Wed, 18 Dec 2024 14:42:44 +0100 Subject: [PATCH 03/13] fix ruff --- .../document-search/basic/evaluate.py | 3 ++- .../document-search/basic/ingest.py | 1 + .../document-search/basic/optimize.py | 26 +++++++++++++------ .../ragbits/core/vector_stores/__init__.py | 6 ++--- .../src/ragbits/evaluate/evaluator.py | 5 ++-- .../src/ragbits/evaluate/optimizer.py | 12 +++++---- .../ragbits/evaluate/pipelines/__init__.py | 7 +++++ .../evaluate/pipelines/document_search.py | 2 ++ 8 files changed, 43 insertions(+), 19 deletions(-) diff --git a/examples/evaluation/document-search/basic/evaluate.py b/examples/evaluation/document-search/basic/evaluate.py index 151da795..5601b414 100644 --- a/examples/evaluation/document-search/basic/evaluate.py +++ b/examples/evaluation/document-search/basic/evaluate.py @@ -13,8 +13,9 @@ from pathlib import Path from omegaconf import OmegaConf -from ragbits.evaluate.utils import log_to_file + from ragbits.evaluate.evaluator import Evaluator +from ragbits.evaluate.utils import log_to_file logging.getLogger("LiteLLM").setLevel(logging.ERROR) logging.getLogger("httpx").setLevel(logging.ERROR) diff --git a/examples/evaluation/document-search/basic/ingest.py b/examples/evaluation/document-search/basic/ingest.py index 77dfff5c..46927f34 100644 --- a/examples/evaluation/document-search/basic/ingest.py +++ b/examples/evaluation/document-search/basic/ingest.py @@ -11,6 +11,7 @@ import logging from omegaconf import OmegaConf + from ragbits.evaluate.pipelines import pipeline_factory logging.getLogger("LiteLLM").setLevel(logging.ERROR) diff --git a/examples/evaluation/document-search/basic/optimize.py b/examples/evaluation/document-search/basic/optimize.py index d835d090..75035344 100644 --- a/examples/evaluation/document-search/basic/optimize.py +++ b/examples/evaluation/document-search/basic/optimize.py @@ -1,6 +1,6 @@ import sys -from omegaconf import DictConfig, OmegaConf +from omegaconf import OmegaConf from ragbits.evaluate.optimizer import Optimizer from ragbits.evaluate.utils import log_optimization_to_file @@ -12,20 +12,30 @@ def main() -> None: """ Function running evaluation for all datasets and evaluation tasks defined in config. """ - config = OmegaConf.create( { "pipeline": { "type": "ragbits.evaluate.pipelines.document_search:DocumentSearchWithIngestionPipeline", "ingest": True, "search": True, + "answer_data_source": { + "name": "hf-docs", + "path": "micpst/hf-docs", + "split": "train", + "num_docs": 5, + }, + "providers": { + "txt": { + "type": "ragbits.document_search.ingestion.providers.unstructured:UnstructuredDefaultProvider" + } + }, "embedder": { - "model": "text-embedding-3-small", - "options": {"dimensions": {"optimize": True, "range": [32, 512]}, "encoding_format": float}, - "providers": { - "txt": { - "type": "ragbits.document_search.ingestion.providers.unstructured:UnstructuredDefaultProvider" - } + "type": "ragbits.core.embeddings.litellm:LiteLLMEmbeddings", + "config": { + "model": "text-embedding-3-small", + "options": { + "dimensions": {"optimize": True, "range": [32, 512]}, + }, }, }, }, diff --git a/packages/ragbits-core/src/ragbits/core/vector_stores/__init__.py b/packages/ragbits-core/src/ragbits/core/vector_stores/__init__.py index e27b6690..426e690a 100644 --- a/packages/ragbits-core/src/ragbits/core/vector_stores/__init__.py +++ b/packages/ragbits-core/src/ragbits/core/vector_stores/__init__.py @@ -1,14 +1,14 @@ from ragbits.core.vector_stores.base import VectorStore, VectorStoreEntry, VectorStoreOptions, WhereQuery -from ragbits.core.vector_stores.in_memory import InMemoryVectorStore from ragbits.core.vector_stores.chroma import ChromaVectorStore +from ragbits.core.vector_stores.in_memory import InMemoryVectorStore from ragbits.core.vector_stores.qdrant import QdrantVectorStore __all__ = [ + "ChromaVectorStore", "InMemoryVectorStore", + "QdrantVectorStore", "VectorStore", "VectorStoreEntry", "VectorStoreOptions", "WhereQuery", - "ChromaVectorStore", - "QdrantVectorStore", ] diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/evaluator.py b/packages/ragbits-evaluate/src/ragbits/evaluate/evaluator.py index 4f7e5db0..79b5d60a 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/evaluator.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/evaluator.py @@ -6,12 +6,13 @@ from omegaconf import DictConfig from tqdm.asyncio import tqdm +from ragbits.evaluate.pipelines.base import EvaluationPipeline, EvaluationResult + from .loaders import dataloader_factory -from .metrics import metric_set_factory from .loaders.base import DataLoader +from .metrics import metric_set_factory from .metrics.base import MetricSet from .pipelines import pipeline_factory -from ragbits.evaluate.pipelines.base import EvaluationPipeline, EvaluationResult class Evaluator: diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/optimizer.py b/packages/ragbits-evaluate/src/ragbits/evaluate/optimizer.py index f730f8c8..35491dfe 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/optimizer.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/optimizer.py @@ -5,18 +5,18 @@ from typing import Any import optuna -from omegaconf import DictConfig, ListConfig, OmegaConf +from omegaconf import DictConfig, ListConfig + from ragbits.core.utils.config_handling import import_by_path -from .loaders import dataloader_factory -from .metrics import metric_set_factory from .callbacks.base import CallbackConfigurator from .evaluator import Evaluator +from .loaders import dataloader_factory from .loaders.base import DataLoader +from .metrics import metric_set_factory from .metrics.base import MetricSet from .pipelines.base import EvaluationPipeline - module = sys.modules[__name__] @@ -34,7 +34,9 @@ def __init__(self, cfg: DictConfig): self._choices_cache: dict[str, list[Any]] = {} @classmethod - def run_experiment_from_config(cls, config: dict[str, DictConfig]) -> list[tuple[DictConfig, float, dict[str, float]]]: + def run_experiment_from_config( + cls, config: dict[str, DictConfig] + ) -> list[tuple[DictConfig, float, dict[str, float]]]: """ Runs the optimization experiment configured with config object Args: diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/__init__.py b/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/__init__.py index cdfb8e02..b7f26a4f 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/__init__.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/__init__.py @@ -10,6 +10,13 @@ def pipeline_factory(pipeline_config: DictConfig) -> EvaluationPipeline: + """ + Factory of evaluation pipelines + Args: + pipeline_config: DictConfig + Returns: + instance of evaluation pipeline + """ pipeline_module = import_by_path(pipeline_config.type, module) pipeline = pipeline_module(pipeline_config) return pipeline diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/document_search.py b/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/document_search.py index 6ce1c89a..b772ae89 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/document_search.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/document_search.py @@ -85,6 +85,8 @@ async def __call__(self, data: dict | None = None) -> DocumentSearchResult | Non return await super().__call__(data) async def _ingest_documents(self) -> None: + if self.config.get("search", False) and self.config.get("ingest", False): + self.document_search.vector_store._index_name = str(uuid.uuid4()) documents = await tqdm.gather( *[ DocumentMeta.from_source( From 5cd3f319027ea17fac3137481c52d4a611afa561 Mon Sep 17 00:00:00 2001 From: kdziedzic68 Date: Wed, 18 Dec 2024 14:57:36 +0100 Subject: [PATCH 04/13] simplify examples --- .../document-search/advanced/optimize.py | 4 ++-- .../basic/{evaluate.py => basic_evaluate.py} | 0 .../basic/{ingest.py => basic_ingest.py} | 0 .../basic/{optimize.py => basic_optimize.py} | 4 ++-- .../src/ragbits/evaluate/optimizer.py | 14 +++++------ .../src/ragbits/evaluate/pipelines/base.py | 2 +- .../evaluate/pipelines/document_search.py | 23 +++++++++---------- 7 files changed, 23 insertions(+), 24 deletions(-) rename examples/evaluation/document-search/basic/{evaluate.py => basic_evaluate.py} (100%) rename examples/evaluation/document-search/basic/{ingest.py => basic_ingest.py} (100%) rename examples/evaluation/document-search/basic/{optimize.py => basic_optimize.py} (93%) diff --git a/examples/evaluation/document-search/advanced/optimize.py b/examples/evaluation/document-search/advanced/optimize.py index 4644f331..24a93aab 100644 --- a/examples/evaluation/document-search/advanced/optimize.py +++ b/examples/evaluation/document-search/advanced/optimize.py @@ -17,8 +17,8 @@ def main(config: DictConfig) -> None: Args: config: Hydra configuration. """ - config = {"optimizer": OmegaConf.create({"direction": "maximize", "n_trials": 10}), "experiment_config": config} - configs_with_scores = Optimizer.run_experiment_from_config(config=config) + exp_config = {"optimizer": OmegaConf.create({"direction": "maximize", "n_trials": 10}), "experiment_config": config} + configs_with_scores = Optimizer.run_experiment_from_config(config=exp_config) log_optimization_to_file(configs_with_scores) diff --git a/examples/evaluation/document-search/basic/evaluate.py b/examples/evaluation/document-search/basic/basic_evaluate.py similarity index 100% rename from examples/evaluation/document-search/basic/evaluate.py rename to examples/evaluation/document-search/basic/basic_evaluate.py diff --git a/examples/evaluation/document-search/basic/ingest.py b/examples/evaluation/document-search/basic/basic_ingest.py similarity index 100% rename from examples/evaluation/document-search/basic/ingest.py rename to examples/evaluation/document-search/basic/basic_ingest.py diff --git a/examples/evaluation/document-search/basic/optimize.py b/examples/evaluation/document-search/basic/basic_optimize.py similarity index 93% rename from examples/evaluation/document-search/basic/optimize.py rename to examples/evaluation/document-search/basic/basic_optimize.py index 75035344..a36abd11 100644 --- a/examples/evaluation/document-search/basic/optimize.py +++ b/examples/evaluation/document-search/basic/basic_optimize.py @@ -52,8 +52,8 @@ def main() -> None: ], } ) - config = {"optimizer": OmegaConf.create({"direction": "maximize", "n_trials": 10}), "experiment_config": config} - configs_with_scores = Optimizer.run_experiment_from_config(config=config) + exp_config = {"optimizer": OmegaConf.create({"direction": "maximize", "n_trials": 10}), "experiment_config": config} + configs_with_scores = Optimizer.run_experiment_from_config(config=exp_config) log_optimization_to_file(configs_with_scores) diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/optimizer.py b/packages/ragbits-evaluate/src/ragbits/evaluate/optimizer.py index 35491dfe..45801f02 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/optimizer.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/optimizer.py @@ -45,20 +45,20 @@ def run_experiment_from_config( list of configs with scores """ optimizer_config = config["optimizer"] - config = config["experiment_config"] - dataloader = dataloader_factory(config.data) - pipeline_class = import_by_path(config.pipeline.type, module) - metrics = metric_set_factory(config.metrics) + exp_config = config["experiment_config"] + dataloader = dataloader_factory(exp_config.data) + pipeline_class = import_by_path(exp_config.pipeline.type, module) + metrics = metric_set_factory(exp_config.metrics) callback_configurators = None - if getattr(config, "callbacks", None): + if getattr(exp_config, "callbacks", None): callback_configurators = [ - import_by_path(callback_cfg.type, module)(callback_cfg.args) for callback_cfg in config.callbacks + import_by_path(callback_cfg.type, module)(callback_cfg.args) for callback_cfg in exp_config.callbacks ] optimizer = cls(cfg=optimizer_config) configs_with_scores = optimizer.optimize( pipeline_class=pipeline_class, - config_with_params=config.pipeline, + config_with_params=exp_config.pipeline, metrics=metrics, dataloader=dataloader, callback_configurators=callback_configurators, diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/base.py b/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/base.py index 9f08a862..2fe0401e 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/base.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/base.py @@ -28,7 +28,7 @@ def __init__(self, config: DictConfig | None = None) -> None: self.config = config or DictConfig({}) @abstractmethod - async def __call__(self, data: dict[str, Any]) -> EvaluationResult: + async def __call__(self, data: dict[str, Any] | None = None) -> EvaluationResult | None: """ Runs the evaluation pipeline. diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/document_search.py b/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/document_search.py index b772ae89..c62364fb 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/document_search.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/document_search.py @@ -1,5 +1,4 @@ import asyncio -import uuid from dataclasses import dataclass from functools import cached_property @@ -38,7 +37,7 @@ def document_search(self) -> "DocumentSearch": """ return DocumentSearch.from_config(self.config) # type: ignore - async def __call__(self, data: dict | None) -> DocumentSearchResult | None: + async def __call__(self, data: dict | None = None) -> DocumentSearchResult | None: """ Runs the document search evaluation pipeline. @@ -48,13 +47,15 @@ async def __call__(self, data: dict | None) -> DocumentSearchResult | None: Returns: The evaluation result. """ - elements = await self.document_search.search(data["question"]) - predicted_passages = [element.text_representation or "" for element in elements] - return DocumentSearchResult( - question=data["question"], - reference_passages=data["passages"], - predicted_passages=predicted_passages, - ) + if data is not None: + elements = await self.document_search.search(data["question"]) + predicted_passages = [element.text_representation or "" for element in elements] + return DocumentSearchResult( + question=data["question"], + reference_passages=data["passages"], + predicted_passages=predicted_passages, + ) + return None class DocumentSearchWithIngestionPipeline(DocumentSearchPipeline): @@ -68,7 +69,7 @@ def __init__(self, config: DictConfig | None = None) -> None: self._ingested = False self._lock = asyncio.Lock() - async def __call__(self, data: dict | None = None) -> DocumentSearchResult | None: + async def __call__(self, data: dict | None = None) -> DocumentSearchResult | None: # type: ignore """ Queries a vector store with given data Ingests the corpus to the store if has not been done @@ -85,8 +86,6 @@ async def __call__(self, data: dict | None = None) -> DocumentSearchResult | Non return await super().__call__(data) async def _ingest_documents(self) -> None: - if self.config.get("search", False) and self.config.get("ingest", False): - self.document_search.vector_store._index_name = str(uuid.uuid4()) documents = await tqdm.gather( *[ DocumentMeta.from_source( From 6e4b2d6dd27b4bd54b0c5c250a1123f27f4f2b75 Mon Sep 17 00:00:00 2001 From: kdziedzic68 Date: Thu, 9 Jan 2025 10:43:40 +0100 Subject: [PATCH 05/13] uncomment --- .../src/ragbits/evaluate/pipelines/document_search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/document_search.py b/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/document_search.py index c62364fb..c4b39149 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/document_search.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/document_search.py @@ -65,7 +65,7 @@ class DocumentSearchWithIngestionPipeline(DocumentSearchPipeline): def __init__(self, config: DictConfig | None = None) -> None: super().__init__(config) - # self.config.vector_store.config.index_name = str(uuid.uuid4()) + self.config.vector_store.config.index_name = str(uuid.uuid4()) self._ingested = False self._lock = asyncio.Lock() From 33a6e1a0cacde39d633749c24282e3c3e7e6ec1a Mon Sep 17 00:00:00 2001 From: kdziedzic68 Date: Thu, 9 Jan 2025 10:45:56 +0100 Subject: [PATCH 06/13] add info about neptune usage in opt --- .../evaluation/document-search/advanced/config/optimization.yaml | 1 + .../src/ragbits/evaluate/pipelines/document_search.py | 1 + 2 files changed, 2 insertions(+) diff --git a/examples/evaluation/document-search/advanced/config/optimization.yaml b/examples/evaluation/document-search/advanced/config/optimization.yaml index 807676cc..c5e7afdc 100644 --- a/examples/evaluation/document-search/advanced/config/optimization.yaml +++ b/examples/evaluation/document-search/advanced/config/optimization.yaml @@ -19,6 +19,7 @@ metrics: threshold: 0.5 +#uncomment to use neptune tracking (set your project name) #callbacks: # - type: ragbits.evaluate.callbacks.neptune:NeptuneCallbackConfigurator # args: diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/document_search.py b/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/document_search.py index c4b39149..cbca57f7 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/document_search.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/document_search.py @@ -1,3 +1,4 @@ +import uuid import asyncio from dataclasses import dataclass from functools import cached_property From 605968e7d5a3033d4e3f0b3fe6b72abef7c71f00 Mon Sep 17 00:00:00 2001 From: kdziedzic68 Date: Thu, 9 Jan 2025 10:53:17 +0100 Subject: [PATCH 07/13] fix ruff --- .../src/ragbits/evaluate/pipelines/document_search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/document_search.py b/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/document_search.py index cbca57f7..3a4fa8a8 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/document_search.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/document_search.py @@ -1,5 +1,5 @@ -import uuid import asyncio +import uuid from dataclasses import dataclass from functools import cached_property From 405fa3572d8cfeb159329ebd406072dd4afebb91 Mon Sep 17 00:00:00 2001 From: kdziedzic68 Date: Thu, 9 Jan 2025 14:16:22 +0100 Subject: [PATCH 08/13] inherit from with construction config --- .../document-search/basic/basic_evaluate.py | 17 +++++++++++++++++ .../document-search/basic/basic_ingest.py | 4 +--- .../document-search/basic/basic_optimize.py | 6 +++++- .../src/ragbits/evaluate/callbacks/base.py | 9 +++++---- .../src/ragbits/evaluate/evaluator.py | 8 ++++---- .../src/ragbits/evaluate/loaders/__init__.py | 6 +++--- .../src/ragbits/evaluate/loaders/base.py | 10 ++++++---- .../src/ragbits/evaluate/metrics/__init__.py | 6 +++--- .../src/ragbits/evaluate/metrics/base.py | 9 +++++---- .../ragbits/evaluate/metrics/document_search.py | 2 +- .../src/ragbits/evaluate/optimizer.py | 12 ++++++------ .../src/ragbits/evaluate/pipelines/__init__.py | 7 ++++--- .../src/ragbits/evaluate/pipelines/base.py | 10 ++++++---- .../evaluate/pipelines/document_search.py | 2 +- 14 files changed, 67 insertions(+), 41 deletions(-) diff --git a/examples/evaluation/document-search/basic/basic_evaluate.py b/examples/evaluation/document-search/basic/basic_evaluate.py index 5601b414..80fce622 100644 --- a/examples/evaluation/document-search/basic/basic_evaluate.py +++ b/examples/evaluation/document-search/basic/basic_evaluate.py @@ -35,6 +35,23 @@ async def evaluate() -> dict: "type": "ragbits.evaluate.pipelines.document_search:DocumentSearchWithIngestionPipeline", "ingest": False, "search": True, + "vector_store": { + "type": "ragbits.core.vector_stores.chroma:ChromaVectorStore", + "config": { + "client": { + "type": "PersistentClient", + "config": { + "path": "chroma" + } + }, + "index_name": "default", + "distance_method": "l2", + "default_options": { + "k": 3, + "max_distance": 1.2 + } + } + }, "providers": { "txt": { "type": "ragbits.document_search.ingestion.providers.unstructured:UnstructuredDefaultProvider" diff --git a/examples/evaluation/document-search/basic/basic_ingest.py b/examples/evaluation/document-search/basic/basic_ingest.py index 46927f34..031cd28a 100644 --- a/examples/evaluation/document-search/basic/basic_ingest.py +++ b/examples/evaluation/document-search/basic/basic_ingest.py @@ -28,8 +28,7 @@ async def ingest() -> None: """ log.info("Ingesting documents...") - config = OmegaConf.create( - { + config = { "type": "ragbits.evaluate.pipelines.document_search:DocumentSearchWithIngestionPipeline", "ingest": True, "search": False, @@ -38,7 +37,6 @@ async def ingest() -> None: "txt": {"type": "ragbits.document_search.ingestion.providers.unstructured:UnstructuredDefaultProvider"} }, } - ) ingestor = pipeline_factory(config) # type: ignore diff --git a/examples/evaluation/document-search/basic/basic_optimize.py b/examples/evaluation/document-search/basic/basic_optimize.py index a36abd11..c8c52878 100644 --- a/examples/evaluation/document-search/basic/basic_optimize.py +++ b/examples/evaluation/document-search/basic/basic_optimize.py @@ -1,4 +1,5 @@ import sys +from pathlib import Path from omegaconf import OmegaConf @@ -54,7 +55,10 @@ def main() -> None: ) exp_config = {"optimizer": OmegaConf.create({"direction": "maximize", "n_trials": 10}), "experiment_config": config} configs_with_scores = Optimizer.run_experiment_from_config(config=exp_config) - log_optimization_to_file(configs_with_scores) + + OUT_DIR = Path("basic_optimization_2") + OUT_DIR.mkdir() + log_optimization_to_file(configs_with_scores, output_dir=OUT_DIR) if __name__ == "__main__": diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/callbacks/base.py b/packages/ragbits-evaluate/src/ragbits/evaluate/callbacks/base.py index 455d50d3..65232f3a 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/callbacks/base.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/callbacks/base.py @@ -1,14 +1,15 @@ from abc import ABC, abstractmethod from collections.abc import Callable -from omegaconf import DictConfig +from omegaconf import OmegaConf +from ragbits.core.utils.config_handling import WithConstructionConfig -class CallbackConfigurator(ABC): +class CallbackConfigurator(WithConstructionConfig, ABC): """An abstract class for callback configuration""" - def __init__(self, config: DictConfig): - self.config = config + def __init__(self, config: dict): + self.config = OmegaConf.create(config) @abstractmethod def get_callback(self) -> Callable: diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/evaluator.py b/packages/ragbits-evaluate/src/ragbits/evaluate/evaluator.py index 79b5d60a..4e996f59 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/evaluator.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/evaluator.py @@ -3,7 +3,7 @@ from dataclasses import asdict from typing import Any -from omegaconf import DictConfig +from omegaconf import DictConfig, OmegaConf from tqdm.asyncio import tqdm from ragbits.evaluate.pipelines.base import EvaluationPipeline, EvaluationResult @@ -57,11 +57,11 @@ async def run_experiment_from_config(cls, config: DictConfig) -> dict[str, Any]: Returns: dictionary of metrics with scores """ - dataloader = dataloader_factory(config.data) - pipeline = pipeline_factory(config.pipeline) + dataloader = dataloader_factory(OmegaConf.to_container(config.data)) + pipeline = pipeline_factory(OmegaConf.to_container(config.pipeline)) metric_config = config.get("metrics", None) - metrics = metric_set_factory(metric_config) if metric_config is not None else None + metrics = metric_set_factory(OmegaConf.to_container(metric_config)) if metric_config is not None else None evaluator = cls() results = await evaluator.compute( diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/loaders/__init__.py b/packages/ragbits-evaluate/src/ragbits/evaluate/loaders/__init__.py index f65da00f..ff4e9eef 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/loaders/__init__.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/loaders/__init__.py @@ -9,7 +9,7 @@ module = sys.modules[__name__] -def dataloader_factory(config: DictConfig) -> DataLoader: +def dataloader_factory(config: dict) -> DataLoader: """ A function creating dataloader from a dataloder config Args: @@ -17,5 +17,5 @@ def dataloader_factory(config: DictConfig) -> DataLoader: Returns: DataLoader """ - dataloader_class = import_by_path(config.type, module) - return dataloader_class(config.options) + dataloader_class = import_by_path(config["type"], module) + return dataloader_class.from_config({"config": config["options"]}) diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/loaders/base.py b/packages/ragbits-evaluate/src/ragbits/evaluate/loaders/base.py index d8407095..a7519dda 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/loaders/base.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/loaders/base.py @@ -1,18 +1,20 @@ from abc import ABC, abstractmethod from typing import Generic, TypeVar -from omegaconf import DictConfig +from omegaconf import DictConfig, OmegaConf + +from ragbits.core.utils.config_handling import WithConstructionConfig DataT = TypeVar("DataT") -class DataLoader(Generic[DataT], ABC): +class DataLoader(WithConstructionConfig, Generic[DataT], ABC): """ Data loader. """ - def __init__(self, config: DictConfig) -> None: - self.config = config + def __init__(self, config: dict) -> None: + self.config = OmegaConf.create(config) @abstractmethod async def load(self) -> DataT: diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/metrics/__init__.py b/packages/ragbits-evaluate/src/ragbits/evaluate/metrics/__init__.py index 408249a6..38b8f141 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/metrics/__init__.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/metrics/__init__.py @@ -9,7 +9,7 @@ module = sys.modules[__name__] -def metric_set_factory(cfg: ListConfig) -> MetricSet: +def metric_set_factory(cfg: list[dict]) -> MetricSet: """ A function creating MetricSet instance from the configuration Args: @@ -19,6 +19,6 @@ def metric_set_factory(cfg: ListConfig) -> MetricSet: """ metrics = [] for metric_cfg in cfg: - metric_module = import_by_path(metric_cfg.type, module) - metrics.append(metric_module(metric_cfg)) + metric_module = import_by_path(metric_cfg["type"], module) + metrics.append(metric_module.from_config({"config": metric_cfg})) return MetricSet(*metrics) diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/metrics/base.py b/packages/ragbits-evaluate/src/ragbits/evaluate/metrics/base.py index a6d5cf70..cc6510f1 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/metrics/base.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/metrics/base.py @@ -1,19 +1,20 @@ from abc import ABC, abstractmethod from typing import Any, Generic, TypeVar -from omegaconf import DictConfig +from omegaconf import DictConfig, OmegaConf +from ragbits.core.utils.config_handling import WithConstructionConfig from ragbits.evaluate.pipelines.base import EvaluationResult ResultT = TypeVar("ResultT", bound=EvaluationResult) -class Metric(Generic[ResultT], ABC): +class Metric(WithConstructionConfig, Generic[ResultT], ABC): """ Base class for metrics. """ - def __init__(self, config: DictConfig | None = None) -> None: + def __init__(self, config: dict | None = None) -> None: """ Initializes the metric. @@ -21,7 +22,7 @@ def __init__(self, config: DictConfig | None = None) -> None: config: The metric configuration. """ super().__init__() - self.config = config + self.config = OmegaConf.create(config) if config else OmegaConf.create({}) self.weight: float = getattr(self.config, "weight", 1.0) @abstractmethod diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/metrics/document_search.py b/packages/ragbits-evaluate/src/ragbits/evaluate/metrics/document_search.py index 9019c2dc..e9591408 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/metrics/document_search.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/metrics/document_search.py @@ -20,7 +20,7 @@ class DocumentSearchMetric(Metric[DocumentSearchResult], ABC): default_matching_strategy: type[RougeChunkMatch] = RougeChunkMatch default_matching_options: DictConfig = OmegaConf.create({"threshold": 0.5}) - def __init__(self, config: DictConfig | None = None) -> None: + def __init__(self, config: dict | None = None) -> None: """ Initializes the metric. diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/optimizer.py b/packages/ragbits-evaluate/src/ragbits/evaluate/optimizer.py index 45801f02..e506684d 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/optimizer.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/optimizer.py @@ -5,7 +5,7 @@ from typing import Any import optuna -from omegaconf import DictConfig, ListConfig +from omegaconf import DictConfig, ListConfig, OmegaConf from ragbits.core.utils.config_handling import import_by_path @@ -46,13 +46,13 @@ def run_experiment_from_config( """ optimizer_config = config["optimizer"] exp_config = config["experiment_config"] - dataloader = dataloader_factory(exp_config.data) + dataloader = dataloader_factory(OmegaConf.to_container(exp_config.data)) pipeline_class = import_by_path(exp_config.pipeline.type, module) - metrics = metric_set_factory(exp_config.metrics) + metrics = metric_set_factory(OmegaConf.to_container(exp_config.metrics)) callback_configurators = None if getattr(exp_config, "callbacks", None): callback_configurators = [ - import_by_path(callback_cfg.type, module)(callback_cfg.args) for callback_cfg in exp_config.callbacks + import_by_path(callback_cfg.type, module)(OmegaConf.to_container(callback_cfg.args)) for callback_cfg in exp_config.callbacks ] optimizer = cls(cfg=optimizer_config) @@ -102,7 +102,7 @@ def objective(trial: optuna.Trial) -> float: study.optimize(objective, **optimization_kwargs) configs_with_scores = [ - (trial.user_attrs["cfg"], trial.user_attrs["score"], trial.user_attrs["all_metrics"]) + (OmegaConf.to_container(trial.user_attrs["cfg"]), trial.user_attrs["score"], trial.user_attrs["all_metrics"]) for trial in study.get_trials() ] @@ -128,7 +128,7 @@ def _objective( try: config_for_trial = deepcopy(config_with_params) self._set_values_for_optimized_params(cfg=config_for_trial, trial=trial, ancestors=[]) - pipeline = pipeline_class(config_for_trial) + pipeline = pipeline_class.from_config(OmegaConf.to_container(config_for_trial)) metrics_values = self._score(pipeline=pipeline, dataloader=dataloader, metrics=metrics) score = sum(metrics_values.values()) break diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/__init__.py b/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/__init__.py index b7f26a4f..d1d5ad96 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/__init__.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/__init__.py @@ -9,7 +9,7 @@ module = sys.modules[__name__] -def pipeline_factory(pipeline_config: DictConfig) -> EvaluationPipeline: +def pipeline_factory(pipeline_config: dict) -> EvaluationPipeline: """ Factory of evaluation pipelines Args: @@ -17,6 +17,7 @@ def pipeline_factory(pipeline_config: DictConfig) -> EvaluationPipeline: Returns: instance of evaluation pipeline """ - pipeline_module = import_by_path(pipeline_config.type, module) - pipeline = pipeline_module(pipeline_config) + pipeline_module = import_by_path(pipeline_config["type"], module) + config = {"config": pipeline_config} + pipeline = pipeline_module.from_config(config) return pipeline diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/base.py b/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/base.py index 2fe0401e..633b3af0 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/base.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/base.py @@ -2,7 +2,8 @@ from dataclasses import dataclass from typing import Any -from omegaconf import DictConfig +from omegaconf import DictConfig, OmegaConf +from ragbits.core.utils.config_handling import WithConstructionConfig @dataclass @@ -12,12 +13,12 @@ class EvaluationResult(ABC): """ -class EvaluationPipeline(ABC): +class EvaluationPipeline(WithConstructionConfig, ABC): """ Collection evaluation pipeline. """ - def __init__(self, config: DictConfig | None = None) -> None: + def __init__(self, config: dict | None) -> None: """ Initializes the evaluation pipeline. @@ -25,7 +26,8 @@ def __init__(self, config: DictConfig | None = None) -> None: config: The evaluation pipeline configuration. """ super().__init__() - self.config = config or DictConfig({}) + + self.config = OmegaConf.create(config) if config else DictConfig({}) @abstractmethod async def __call__(self, data: dict[str, Any] | None = None) -> EvaluationResult | None: diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/document_search.py b/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/document_search.py index 3a4fa8a8..12f23aa3 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/document_search.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/document_search.py @@ -64,7 +64,7 @@ class DocumentSearchWithIngestionPipeline(DocumentSearchPipeline): A class for joint doument ingestion and search """ - def __init__(self, config: DictConfig | None = None) -> None: + def __init__(self, config: dict | None) -> None: super().__init__(config) self.config.vector_store.config.index_name = str(uuid.uuid4()) self._ingested = False From 02e1c94542547d918bf37dea61a033c13c7c1d52 Mon Sep 17 00:00:00 2001 From: kdziedzic68 Date: Thu, 9 Jan 2025 14:56:44 +0100 Subject: [PATCH 09/13] fix linters --- .../document-search/basic/basic_evaluate.py | 14 +++-------- .../document-search/basic/basic_ingest.py | 18 ++++++------- .../document-search/basic/basic_optimize.py | 2 +- .../src/ragbits/evaluate/callbacks/base.py | 1 + .../src/ragbits/evaluate/evaluator.py | 12 ++++++--- .../src/ragbits/evaluate/loaders/__init__.py | 2 -- .../src/ragbits/evaluate/loaders/base.py | 2 +- .../src/ragbits/evaluate/metrics/__init__.py | 2 -- .../src/ragbits/evaluate/metrics/base.py | 2 +- .../src/ragbits/evaluate/optimizer.py | 25 +++++++++++-------- .../ragbits/evaluate/pipelines/__init__.py | 2 -- .../src/ragbits/evaluate/pipelines/base.py | 1 + .../evaluate/pipelines/document_search.py | 1 - .../src/ragbits/evaluate/utils.py | 2 +- 14 files changed, 39 insertions(+), 47 deletions(-) diff --git a/examples/evaluation/document-search/basic/basic_evaluate.py b/examples/evaluation/document-search/basic/basic_evaluate.py index 80fce622..5cda1db5 100644 --- a/examples/evaluation/document-search/basic/basic_evaluate.py +++ b/examples/evaluation/document-search/basic/basic_evaluate.py @@ -38,19 +38,11 @@ async def evaluate() -> dict: "vector_store": { "type": "ragbits.core.vector_stores.chroma:ChromaVectorStore", "config": { - "client": { - "type": "PersistentClient", - "config": { - "path": "chroma" - } - }, + "client": {"type": "PersistentClient", "config": {"path": "chroma"}}, "index_name": "default", "distance_method": "l2", - "default_options": { - "k": 3, - "max_distance": 1.2 - } - } + "default_options": {"k": 3, "max_distance": 1.2}, + }, }, "providers": { "txt": { diff --git a/examples/evaluation/document-search/basic/basic_ingest.py b/examples/evaluation/document-search/basic/basic_ingest.py index 031cd28a..ac41ee87 100644 --- a/examples/evaluation/document-search/basic/basic_ingest.py +++ b/examples/evaluation/document-search/basic/basic_ingest.py @@ -10,8 +10,6 @@ import asyncio import logging -from omegaconf import OmegaConf - from ragbits.evaluate.pipelines import pipeline_factory logging.getLogger("LiteLLM").setLevel(logging.ERROR) @@ -29,14 +27,14 @@ async def ingest() -> None: log.info("Ingesting documents...") config = { - "type": "ragbits.evaluate.pipelines.document_search:DocumentSearchWithIngestionPipeline", - "ingest": True, - "search": False, - "answer_data_source": {"name": "hf-docs", "path": "micpst/hf-docs", "split": "train", "num_docs": 5}, - "providers": { - "txt": {"type": "ragbits.document_search.ingestion.providers.unstructured:UnstructuredDefaultProvider"} - }, - } + "type": "ragbits.evaluate.pipelines.document_search:DocumentSearchWithIngestionPipeline", + "ingest": True, + "search": False, + "answer_data_source": {"name": "hf-docs", "path": "micpst/hf-docs", "split": "train", "num_docs": 5}, + "providers": { + "txt": {"type": "ragbits.document_search.ingestion.providers.unstructured:UnstructuredDefaultProvider"} + }, + } ingestor = pipeline_factory(config) # type: ignore diff --git a/examples/evaluation/document-search/basic/basic_optimize.py b/examples/evaluation/document-search/basic/basic_optimize.py index c8c52878..723eae97 100644 --- a/examples/evaluation/document-search/basic/basic_optimize.py +++ b/examples/evaluation/document-search/basic/basic_optimize.py @@ -56,7 +56,7 @@ def main() -> None: exp_config = {"optimizer": OmegaConf.create({"direction": "maximize", "n_trials": 10}), "experiment_config": config} configs_with_scores = Optimizer.run_experiment_from_config(config=exp_config) - OUT_DIR = Path("basic_optimization_2") + OUT_DIR = Path("basic_optimization") OUT_DIR.mkdir() log_optimization_to_file(configs_with_scores, output_dir=OUT_DIR) diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/callbacks/base.py b/packages/ragbits-evaluate/src/ragbits/evaluate/callbacks/base.py index 65232f3a..95d3f3b1 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/callbacks/base.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/callbacks/base.py @@ -2,6 +2,7 @@ from collections.abc import Callable from omegaconf import OmegaConf + from ragbits.core.utils.config_handling import WithConstructionConfig diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/evaluator.py b/packages/ragbits-evaluate/src/ragbits/evaluate/evaluator.py index 4e996f59..06072321 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/evaluator.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/evaluator.py @@ -1,7 +1,7 @@ import time from collections.abc import Iterable from dataclasses import asdict -from typing import Any +from typing import Any, cast from omegaconf import DictConfig, OmegaConf from tqdm.asyncio import tqdm @@ -57,11 +57,15 @@ async def run_experiment_from_config(cls, config: DictConfig) -> dict[str, Any]: Returns: dictionary of metrics with scores """ - dataloader = dataloader_factory(OmegaConf.to_container(config.data)) - pipeline = pipeline_factory(OmegaConf.to_container(config.pipeline)) + dataloader = dataloader_factory(cast(dict, OmegaConf.to_container(config.data))) + pipeline = pipeline_factory(cast(dict, OmegaConf.to_container(config.pipeline))) metric_config = config.get("metrics", None) - metrics = metric_set_factory(OmegaConf.to_container(metric_config)) if metric_config is not None else None + metrics = ( + metric_set_factory(cast(list[dict], OmegaConf.to_container(metric_config))) + if metric_config is not None + else None + ) evaluator = cls() results = await evaluator.compute( diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/loaders/__init__.py b/packages/ragbits-evaluate/src/ragbits/evaluate/loaders/__init__.py index ff4e9eef..0fc20043 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/loaders/__init__.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/loaders/__init__.py @@ -1,7 +1,5 @@ import sys -from omegaconf import DictConfig - from ragbits.core.utils.config_handling import import_by_path from .base import DataLoader diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/loaders/base.py b/packages/ragbits-evaluate/src/ragbits/evaluate/loaders/base.py index a7519dda..b8a458ce 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/loaders/base.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/loaders/base.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod from typing import Generic, TypeVar -from omegaconf import DictConfig, OmegaConf +from omegaconf import OmegaConf from ragbits.core.utils.config_handling import WithConstructionConfig diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/metrics/__init__.py b/packages/ragbits-evaluate/src/ragbits/evaluate/metrics/__init__.py index 38b8f141..505d4b2b 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/metrics/__init__.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/metrics/__init__.py @@ -1,7 +1,5 @@ import sys -from omegaconf import ListConfig - from ragbits.core.utils.config_handling import import_by_path from .base import MetricSet diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/metrics/base.py b/packages/ragbits-evaluate/src/ragbits/evaluate/metrics/base.py index cc6510f1..e3195d76 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/metrics/base.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/metrics/base.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod from typing import Any, Generic, TypeVar -from omegaconf import DictConfig, OmegaConf +from omegaconf import OmegaConf from ragbits.core.utils.config_handling import WithConstructionConfig from ragbits.evaluate.pipelines.base import EvaluationResult diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/optimizer.py b/packages/ragbits-evaluate/src/ragbits/evaluate/optimizer.py index e506684d..83a1f221 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/optimizer.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/optimizer.py @@ -2,7 +2,7 @@ import sys import warnings from copy import deepcopy -from typing import Any +from typing import Any, cast import optuna from omegaconf import DictConfig, ListConfig, OmegaConf @@ -34,9 +34,7 @@ def __init__(self, cfg: DictConfig): self._choices_cache: dict[str, list[Any]] = {} @classmethod - def run_experiment_from_config( - cls, config: dict[str, DictConfig] - ) -> list[tuple[DictConfig, float, dict[str, float]]]: + def run_experiment_from_config(cls, config: dict[str, DictConfig]) -> list[tuple[dict, float, dict[str, float]]]: """ Runs the optimization experiment configured with config object Args: @@ -46,13 +44,14 @@ def run_experiment_from_config( """ optimizer_config = config["optimizer"] exp_config = config["experiment_config"] - dataloader = dataloader_factory(OmegaConf.to_container(exp_config.data)) + dataloader = dataloader_factory(cast(dict, OmegaConf.to_container(exp_config.data))) pipeline_class = import_by_path(exp_config.pipeline.type, module) - metrics = metric_set_factory(OmegaConf.to_container(exp_config.metrics)) + metrics = metric_set_factory(cast(list[dict], OmegaConf.to_container(exp_config.metrics))) callback_configurators = None if getattr(exp_config, "callbacks", None): callback_configurators = [ - import_by_path(callback_cfg.type, module)(OmegaConf.to_container(callback_cfg.args)) for callback_cfg in exp_config.callbacks + import_by_path(callback_cfg.type, module)(OmegaConf.to_container(callback_cfg.args)) + for callback_cfg in exp_config.callbacks ] optimizer = cls(cfg=optimizer_config) @@ -72,7 +71,7 @@ def optimize( dataloader: DataLoader, metrics: MetricSet, callback_configurators: list[CallbackConfigurator] | None = None, - ) -> list[tuple[DictConfig, float, dict[str, float]]]: + ) -> list[tuple[dict, float, dict[str, float]]]: """ A method for running the optimization process for given parameters Args: @@ -102,11 +101,15 @@ def objective(trial: optuna.Trial) -> float: study.optimize(objective, **optimization_kwargs) configs_with_scores = [ - (OmegaConf.to_container(trial.user_attrs["cfg"]), trial.user_attrs["score"], trial.user_attrs["all_metrics"]) + ( + cast(dict, OmegaConf.to_container(trial.user_attrs["cfg"])), + trial.user_attrs["score"], + trial.user_attrs["all_metrics"], + ) for trial in study.get_trials() ] - def sorting_key(results: tuple[DictConfig, float, dict[str, float]]) -> float: + def sorting_key(results: tuple[dict, float, dict[str, float]]) -> float: if self.config.direction == "maximize": return -results[1] else: @@ -128,7 +131,7 @@ def _objective( try: config_for_trial = deepcopy(config_with_params) self._set_values_for_optimized_params(cfg=config_for_trial, trial=trial, ancestors=[]) - pipeline = pipeline_class.from_config(OmegaConf.to_container(config_for_trial)) + pipeline = pipeline_class.from_config({"config": OmegaConf.to_container(config_for_trial)}) metrics_values = self._score(pipeline=pipeline, dataloader=dataloader, metrics=metrics) score = sum(metrics_values.values()) break diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/__init__.py b/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/__init__.py index d1d5ad96..21e985a4 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/__init__.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/__init__.py @@ -1,7 +1,5 @@ import sys -from omegaconf import DictConfig - from ragbits.core.utils.config_handling import import_by_path from .base import EvaluationPipeline diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/base.py b/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/base.py index 633b3af0..2a382f4b 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/base.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/base.py @@ -3,6 +3,7 @@ from typing import Any from omegaconf import DictConfig, OmegaConf + from ragbits.core.utils.config_handling import WithConstructionConfig diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/document_search.py b/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/document_search.py index 12f23aa3..9908fc47 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/document_search.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/document_search.py @@ -3,7 +3,6 @@ from dataclasses import dataclass from functools import cached_property -from omegaconf import DictConfig from tqdm.asyncio import tqdm from ragbits.document_search import DocumentSearch diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/utils.py b/packages/ragbits-evaluate/src/ragbits/evaluate/utils.py index b24883a8..87129acd 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/utils.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/utils.py @@ -68,7 +68,7 @@ def log_dataset_to_file(dataset: Dataset, output_dir: Path | None = None) -> Pat def log_optimization_to_file( - results: list[tuple[DictConfig, float, dict[str, float]]], output_dir: Path | None = None + results: list[tuple[dict, float, dict[str, float]]], output_dir: Path | None = None ) -> Path: """ Log the evaluation results locally. From 80173a46344651633dd6c27eb7258dab79ea42e4 Mon Sep 17 00:00:00 2001 From: kdziedzic68 Date: Thu, 9 Jan 2025 15:13:34 +0100 Subject: [PATCH 10/13] fix linters --- .../document-search/advanced/optimize.py | 6 +- .../document-search/basic/basic_optimize.py | 74 +++++++++---------- .../src/ragbits/core/prompt/promptfoo.py | 2 +- .../core/utils/dict_transformations.py | 2 +- .../evaluate/dataset_generator/prompts/qa.py | 6 +- .../src/ragbits/evaluate/optimizer.py | 17 +++-- 6 files changed, 52 insertions(+), 55 deletions(-) diff --git a/examples/evaluation/document-search/advanced/optimize.py b/examples/evaluation/document-search/advanced/optimize.py index 24a93aab..38a706dd 100644 --- a/examples/evaluation/document-search/advanced/optimize.py +++ b/examples/evaluation/document-search/advanced/optimize.py @@ -1,4 +1,5 @@ import sys +from typing import cast import hydra from omegaconf import DictConfig, OmegaConf @@ -17,7 +18,10 @@ def main(config: DictConfig) -> None: Args: config: Hydra configuration. """ - exp_config = {"optimizer": OmegaConf.create({"direction": "maximize", "n_trials": 10}), "experiment_config": config} + exp_config = { + "optimizer": {"direction": "maximize", "n_trials": 10}, + "experiment_config": cast(dict, OmegaConf.to_container(config)), + } configs_with_scores = Optimizer.run_experiment_from_config(config=exp_config) log_optimization_to_file(configs_with_scores) diff --git a/examples/evaluation/document-search/basic/basic_optimize.py b/examples/evaluation/document-search/basic/basic_optimize.py index 723eae97..adaf4953 100644 --- a/examples/evaluation/document-search/basic/basic_optimize.py +++ b/examples/evaluation/document-search/basic/basic_optimize.py @@ -1,8 +1,6 @@ import sys from pathlib import Path -from omegaconf import OmegaConf - from ragbits.evaluate.optimizer import Optimizer from ragbits.evaluate.utils import log_optimization_to_file @@ -13,47 +11,43 @@ def main() -> None: """ Function running evaluation for all datasets and evaluation tasks defined in config. """ - config = OmegaConf.create( - { - "pipeline": { - "type": "ragbits.evaluate.pipelines.document_search:DocumentSearchWithIngestionPipeline", - "ingest": True, - "search": True, - "answer_data_source": { - "name": "hf-docs", - "path": "micpst/hf-docs", - "split": "train", - "num_docs": 5, - }, - "providers": { - "txt": { - "type": "ragbits.document_search.ingestion.providers.unstructured:UnstructuredDefaultProvider" - } - }, - "embedder": { - "type": "ragbits.core.embeddings.litellm:LiteLLMEmbeddings", - "config": { - "model": "text-embedding-3-small", - "options": { - "dimensions": {"optimize": True, "range": [32, 512]}, - }, + config = { + "pipeline": { + "type": "ragbits.evaluate.pipelines.document_search:DocumentSearchWithIngestionPipeline", + "ingest": True, + "search": True, + "answer_data_source": { + "name": "hf-docs", + "path": "micpst/hf-docs", + "split": "train", + "num_docs": 5, + }, + "providers": { + "txt": {"type": "ragbits.document_search.ingestion.providers.unstructured:UnstructuredDefaultProvider"} + }, + "embedder": { + "type": "ragbits.core.embeddings.litellm:LiteLLMEmbeddings", + "config": { + "model": "text-embedding-3-small", + "options": { + "dimensions": {"optimize": True, "range": [32, 512]}, }, }, }, - "data": { - "type": "ragbits.evaluate.loaders.hf:HFDataLoader", - "options": {"name": "hf-docs-retrieval", "path": "micpst/hf-docs-retrieval", "split": "train"}, - }, - "metrics": [ - { - "type": "ragbits.evaluate.metrics.document_search:DocumentSearchPrecisionRecallF1", - "matching_strategy": "RougeChunkMatch", - "options": {"threshold": 0.5}, - } - ], - } - ) - exp_config = {"optimizer": OmegaConf.create({"direction": "maximize", "n_trials": 10}), "experiment_config": config} + }, + "data": { + "type": "ragbits.evaluate.loaders.hf:HFDataLoader", + "options": {"name": "hf-docs-retrieval", "path": "micpst/hf-docs-retrieval", "split": "train"}, + }, + "metrics": [ + { + "type": "ragbits.evaluate.metrics.document_search:DocumentSearchPrecisionRecallF1", + "matching_strategy": "RougeChunkMatch", + "options": {"threshold": 0.5}, + } + ], + } + exp_config = {"optimizer": {"direction": "maximize", "n_trials": 10}, "experiment_config": config} configs_with_scores = Optimizer.run_experiment_from_config(config=exp_config) OUT_DIR = Path("basic_optimization") diff --git a/packages/ragbits-core/src/ragbits/core/prompt/promptfoo.py b/packages/ragbits-core/src/ragbits/core/prompt/promptfoo.py index 73d5f679..ea9a4193 100644 --- a/packages/ragbits-core/src/ragbits/core/prompt/promptfoo.py +++ b/packages/ragbits-core/src/ragbits/core/prompt/promptfoo.py @@ -46,5 +46,5 @@ def generate_configs( target_path.mkdir() for prompt in prompts: with open(target_path / f"{prompt.__qualname__}.yaml", "w", encoding="utf-8") as f: - prompt_path = f'file://{prompt.__module__.replace(".", os.sep)}.py:{prompt.__qualname__}.to_promptfoo' + prompt_path = f"file://{prompt.__module__.replace('.', os.sep)}.py:{prompt.__qualname__}.to_promptfoo" yaml.dump({"prompts": [prompt_path]}, f) diff --git a/packages/ragbits-core/src/ragbits/core/utils/dict_transformations.py b/packages/ragbits-core/src/ragbits/core/utils/dict_transformations.py index ae58fc62..617cce76 100644 --- a/packages/ragbits-core/src/ragbits/core/utils/dict_transformations.py +++ b/packages/ragbits-core/src/ragbits/core/utils/dict_transformations.py @@ -98,7 +98,7 @@ def _decompose_key(key: str) -> tuple[str | int | None, str | int | None]: _current_subkey = int(_key[start_subscript_index:end_subscript_index]) if len(_key[end_subscript_index:]) > 1: - _current_subkey = f"{_current_subkey}.{_key[end_subscript_index + 2:]}" + _current_subkey = f"{_current_subkey}.{_key[end_subscript_index + 2 :]}" break elif char == ".": split_work = _key.split(".", 1) diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/dataset_generator/prompts/qa.py b/packages/ragbits-evaluate/src/ragbits/evaluate/dataset_generator/prompts/qa.py index 40e29078..d9295ae3 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/dataset_generator/prompts/qa.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/dataset_generator/prompts/qa.py @@ -23,7 +23,7 @@ class BasicAnswerGenPrompt(Prompt[BasicAnswerGenInput, str]): "If you don't know the answer just say: I don't know." ) - user_prompt: str = "Text:\n<|text_start|>\n {{ chunk }} \n<|text_end|>\n\nQuestion:\n " "{{ question }} \n\nAnswer:" + user_prompt: str = "Text:\n<|text_start|>\n {{ chunk }} \n<|text_end|>\n\nQuestion:\n {{ question }} \n\nAnswer:" class PassagesGenInput(BaseModel): @@ -49,9 +49,7 @@ class PassagesGenPrompt(Prompt[PassagesGenInput, str]): "FULL SENTENCES" ) - user_prompt: str = ( - "Question:\n {{ question }} \nAnswer:\n {{ basic_answer }} \nChunk:\n " "{{ chunk }}\n\nPassages:" - ) + user_prompt: str = "Question:\n {{ question }} \nAnswer:\n {{ basic_answer }} \nChunk:\n {{ chunk }}\n\nPassages:" class QueryGenInput(BaseModel): diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/optimizer.py b/packages/ragbits-evaluate/src/ragbits/evaluate/optimizer.py index 83a1f221..29ed0acf 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/optimizer.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/optimizer.py @@ -7,7 +7,7 @@ import optuna from omegaconf import DictConfig, ListConfig, OmegaConf -from ragbits.core.utils.config_handling import import_by_path +from ragbits.core.utils.config_handling import WithConstructionConfig, import_by_path from .callbacks.base import CallbackConfigurator from .evaluator import Evaluator @@ -20,21 +20,21 @@ module = sys.modules[__name__] -class Optimizer: +class Optimizer(WithConstructionConfig): """ Class for optimization """ INFINITY = 1e16 - def __init__(self, cfg: DictConfig): - self.config = cfg + def __init__(self, config: dict): + self.config = OmegaConf.create(config) # workaround for optuna not allowing different choices for different trials # TODO check how optuna handles parallelism. discuss if we want to have parallel studies self._choices_cache: dict[str, list[Any]] = {} @classmethod - def run_experiment_from_config(cls, config: dict[str, DictConfig]) -> list[tuple[dict, float, dict[str, float]]]: + def run_experiment_from_config(cls, config: dict[str, dict]) -> list[tuple[dict, float, dict[str, float]]]: """ Runs the optimization experiment configured with config object Args: @@ -42,8 +42,8 @@ def run_experiment_from_config(cls, config: dict[str, DictConfig]) -> list[tuple Returns: list of configs with scores """ - optimizer_config = config["optimizer"] - exp_config = config["experiment_config"] + optimizer_config = OmegaConf.create(config["optimizer"]) + exp_config = OmegaConf.create(config["experiment_config"]) dataloader = dataloader_factory(cast(dict, OmegaConf.to_container(exp_config.data))) pipeline_class = import_by_path(exp_config.pipeline.type, module) metrics = metric_set_factory(cast(list[dict], OmegaConf.to_container(exp_config.metrics))) @@ -54,7 +54,8 @@ def run_experiment_from_config(cls, config: dict[str, DictConfig]) -> list[tuple for callback_cfg in exp_config.callbacks ] - optimizer = cls(cfg=optimizer_config) + optimizer_config_dict = {"config": OmegaConf.to_container(optimizer_config)} + optimizer = cls.from_config(optimizer_config_dict) configs_with_scores = optimizer.optimize( pipeline_class=pipeline_class, config_with_params=exp_config.pipeline, From 18c967567caac444a87fb33de6ceed8c93d47c6a Mon Sep 17 00:00:00 2001 From: kdziedzic68 Date: Thu, 9 Jan 2025 15:27:11 +0100 Subject: [PATCH 11/13] fixes --- .../document-search/advanced/evaluate.py | 5 +- .../document-search/basic/basic_evaluate.py | 64 +++++++++---------- .../src/ragbits/evaluate/evaluator.py | 8 +-- 3 files changed, 36 insertions(+), 41 deletions(-) diff --git a/examples/evaluation/document-search/advanced/evaluate.py b/examples/evaluation/document-search/advanced/evaluate.py index 456e08b0..4bd6671b 100644 --- a/examples/evaluation/document-search/advanced/evaluate.py +++ b/examples/evaluation/document-search/advanced/evaluate.py @@ -8,9 +8,10 @@ # /// import asyncio import logging +from typing import cast import hydra -from omegaconf import DictConfig +from omegaconf import DictConfig, OmegaConf from ragbits.evaluate.evaluator import Evaluator from ragbits.evaluate.utils import log_to_file, log_to_neptune, setup_neptune @@ -29,7 +30,7 @@ async def bench(config: DictConfig) -> None: """ run = setup_neptune(config) log.info("Starting the experiment...") - results = await Evaluator.run_experiment_from_config(config=config) + results = await Evaluator.run_experiment_from_config(config=cast(dict, OmegaConf.to_container(config))) output_dir = log_to_file(results) if run: log_to_neptune(run, results, output_dir) diff --git a/examples/evaluation/document-search/basic/basic_evaluate.py b/examples/evaluation/document-search/basic/basic_evaluate.py index 5cda1db5..f59b1fab 100644 --- a/examples/evaluation/document-search/basic/basic_evaluate.py +++ b/examples/evaluation/document-search/basic/basic_evaluate.py @@ -12,8 +12,6 @@ import uuid from pathlib import Path -from omegaconf import OmegaConf - from ragbits.evaluate.evaluator import Evaluator from ragbits.evaluate.utils import log_to_file @@ -29,42 +27,38 @@ async def evaluate() -> dict: """ log.info("Ingesting documents...") - config = OmegaConf.create( - { - "pipeline": { - "type": "ragbits.evaluate.pipelines.document_search:DocumentSearchWithIngestionPipeline", - "ingest": False, - "search": True, - "vector_store": { - "type": "ragbits.core.vector_stores.chroma:ChromaVectorStore", - "config": { - "client": {"type": "PersistentClient", "config": {"path": "chroma"}}, - "index_name": "default", - "distance_method": "l2", - "default_options": {"k": 3, "max_distance": 1.2}, - }, - }, - "providers": { - "txt": { - "type": "ragbits.document_search.ingestion.providers.unstructured:UnstructuredDefaultProvider" - } + config = { + "pipeline": { + "type": "ragbits.evaluate.pipelines.document_search:DocumentSearchWithIngestionPipeline", + "ingest": False, + "search": True, + "vector_store": { + "type": "ragbits.core.vector_stores.chroma:ChromaVectorStore", + "config": { + "client": {"type": "PersistentClient", "config": {"path": "chroma"}}, + "index_name": "default", + "distance_method": "l2", + "default_options": {"k": 3, "max_distance": 1.2}, }, }, - "data": { - "type": "ragbits.evaluate.loaders.hf:HFDataLoader", - "options": {"name": "hf-docs-retrieval", "path": "micpst/hf-docs-retrieval", "split": "train"}, + "providers": { + "txt": {"type": "ragbits.document_search.ingestion.providers.unstructured:UnstructuredDefaultProvider"} }, - "metrics": [ - { - "type": "ragbits.evaluate.metrics.document_search:DocumentSearchPrecisionRecallF1", - "matching_strategy": "RougeChunkMatch", - "options": {"threshold": 0.5}, - } - ], - "neptune": {"project": "ragbits", "run": False}, - "task": {"name": "default", "type": "document-search"}, - } - ) + }, + "data": { + "type": "ragbits.evaluate.loaders.hf:HFDataLoader", + "options": {"name": "hf-docs-retrieval", "path": "micpst/hf-docs-retrieval", "split": "train"}, + }, + "metrics": [ + { + "type": "ragbits.evaluate.metrics.document_search:DocumentSearchPrecisionRecallF1", + "matching_strategy": "RougeChunkMatch", + "options": {"threshold": 0.5}, + } + ], + "neptune": {"project": "ragbits", "run": False}, + "task": {"name": "default", "type": "document-search"}, + } results = await Evaluator.run_experiment_from_config(config=config) diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/evaluator.py b/packages/ragbits-evaluate/src/ragbits/evaluate/evaluator.py index 06072321..a8d9dfae 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/evaluator.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/evaluator.py @@ -49,7 +49,7 @@ async def compute( } @classmethod - async def run_experiment_from_config(cls, config: DictConfig) -> dict[str, Any]: + async def run_experiment_from_config(cls, config: dict) -> dict[str, Any]: """ Runs the evaluation experiment basing on configuration Args: @@ -57,12 +57,12 @@ async def run_experiment_from_config(cls, config: DictConfig) -> dict[str, Any]: Returns: dictionary of metrics with scores """ - dataloader = dataloader_factory(cast(dict, OmegaConf.to_container(config.data))) - pipeline = pipeline_factory(cast(dict, OmegaConf.to_container(config.pipeline))) + dataloader = dataloader_factory(config["data"]) + pipeline = pipeline_factory(config["pipeline"]) metric_config = config.get("metrics", None) metrics = ( - metric_set_factory(cast(list[dict], OmegaConf.to_container(metric_config))) + metric_set_factory(metric_config) # type: ignore if metric_config is not None else None ) From 9ec00ae66ed30a990e62d05542900da95b19eb89 Mon Sep 17 00:00:00 2001 From: kdziedzic68 Date: Thu, 9 Jan 2025 20:23:40 +0100 Subject: [PATCH 12/13] ruff formatting --- packages/ragbits-evaluate/src/ragbits/evaluate/evaluator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/evaluator.py b/packages/ragbits-evaluate/src/ragbits/evaluate/evaluator.py index a8d9dfae..ec6ff4db 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/evaluator.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/evaluator.py @@ -62,7 +62,7 @@ async def run_experiment_from_config(cls, config: dict) -> dict[str, Any]: metric_config = config.get("metrics", None) metrics = ( - metric_set_factory(metric_config) # type: ignore + metric_set_factory(metric_config) # type: ignore if metric_config is not None else None ) From 7bcf23fe2b438417aa9c246ad3d92f52d4bba738 Mon Sep 17 00:00:00 2001 From: kdziedzic68 Date: Thu, 9 Jan 2025 22:51:59 +0100 Subject: [PATCH 13/13] fix ruff linter --- packages/ragbits-evaluate/src/ragbits/evaluate/evaluator.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/evaluator.py b/packages/ragbits-evaluate/src/ragbits/evaluate/evaluator.py index ec6ff4db..d63e587b 100644 --- a/packages/ragbits-evaluate/src/ragbits/evaluate/evaluator.py +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/evaluator.py @@ -1,9 +1,8 @@ import time from collections.abc import Iterable from dataclasses import asdict -from typing import Any, cast +from typing import Any -from omegaconf import DictConfig, OmegaConf from tqdm.asyncio import tqdm from ragbits.evaluate.pipelines.base import EvaluationPipeline, EvaluationResult @@ -60,7 +59,7 @@ async def run_experiment_from_config(cls, config: dict) -> dict[str, Any]: dataloader = dataloader_factory(config["data"]) pipeline = pipeline_factory(config["pipeline"]) - metric_config = config.get("metrics", None) + metric_config = config.get("metrics") metrics = ( metric_set_factory(metric_config) # type: ignore if metric_config is not None