From 7251935d69626314d82680b8dc5dd1d2c2eb1137 Mon Sep 17 00:00:00 2001 From: Aliaksandr Kuzmik <98702584+alexkuzmik@users.noreply.github.com> Date: Tue, 17 Dec 2024 19:21:45 +0100 Subject: [PATCH] [OPIK-571] error tracking sdk add e 2 e tests for traces spans with errors (#907) * Implement e2e tests for error_info * Fix lint errors --- sdks/python/tests/e2e/test_tracing.py | 138 ++++++++++++++++++++++++++ sdks/python/tests/e2e/verifiers.py | 23 ++++- 2 files changed, 157 insertions(+), 4 deletions(-) diff --git a/sdks/python/tests/e2e/test_tracing.py b/sdks/python/tests/e2e/test_tracing.py index 7476ae84dd..d6d68aecf6 100644 --- a/sdks/python/tests/e2e/test_tracing.py +++ b/sdks/python/tests/e2e/test_tracing.py @@ -7,6 +7,8 @@ from . import verifiers from .conftest import OPIK_E2E_TESTS_PROJECT_NAME +from ..testlib import ANY_STRING + @pytest.mark.parametrize( "project_name", @@ -85,6 +87,142 @@ def f_inner(y): ) +def test_tracked_function__error_inside_inner_function__caugth_in_top_level_span__inner_span_has_error_info( + opik_client, +): + # Setup + ID_STORAGE = {} + + @opik.track + def f_inner(y): + ID_STORAGE["f_inner-span-id"] = opik_context.get_current_span_data().id + raise ValueError("inner span error message") + + @opik.track + def f_outer(x): + ID_STORAGE["f_outer-trace-id"] = opik_context.get_current_trace_data().id + ID_STORAGE["f_outer-span-id"] = opik_context.get_current_span_data().id + + try: + f_inner("inner-input") + except Exception: + pass + + return "outer-output" + + # Call + f_outer("outer-input") + opik.flush_tracker() + + # Verify trace + verifiers.verify_trace( + opik_client=opik_client, + trace_id=ID_STORAGE["f_outer-trace-id"], + name="f_outer", + input={"x": "outer-input"}, + output={"output": "outer-output"}, + error_info=None, + ) + + # Verify top level span + verifiers.verify_span( + opik_client=opik_client, + span_id=ID_STORAGE["f_outer-span-id"], + parent_span_id=None, + trace_id=ID_STORAGE["f_outer-trace-id"], + name="f_outer", + input={"x": "outer-input"}, + output={"output": "outer-output"}, + ) + + # Verify nested span + verifiers.verify_span( + opik_client=opik_client, + span_id=ID_STORAGE["f_inner-span-id"], + parent_span_id=ID_STORAGE["f_outer-span-id"], + trace_id=ID_STORAGE["f_outer-trace-id"], + name="f_inner", + input={"y": "inner-input"}, + output=None, + error_info={ + "exception_type": "ValueError", + "message": "inner span error message", + "traceback": ANY_STRING(), + }, + ) + + +def test_tracked_function__error_inside_inner_function__error_not_caugth__trace_and_its_spans_have_error_info( + opik_client, +): + # Setup + ID_STORAGE = {} + + @opik.track + def f_inner(y): + ID_STORAGE["f_inner-span-id"] = opik_context.get_current_span_data().id + raise ValueError("inner span error message") + + @opik.track + def f_outer(x): + ID_STORAGE["f_outer-trace-id"] = opik_context.get_current_trace_data().id + ID_STORAGE["f_outer-span-id"] = opik_context.get_current_span_data().id + + f_inner("inner-input") + + # Call + with pytest.raises(ValueError): + f_outer("outer-input") + + opik.flush_tracker() + + # Verify trace + verifiers.verify_trace( + opik_client=opik_client, + trace_id=ID_STORAGE["f_outer-trace-id"], + name="f_outer", + input={"x": "outer-input"}, + output=None, + error_info={ + "exception_type": "ValueError", + "message": "inner span error message", + "traceback": ANY_STRING(), + }, + ) + + # Verify top level span + verifiers.verify_span( + opik_client=opik_client, + span_id=ID_STORAGE["f_outer-span-id"], + parent_span_id=None, + trace_id=ID_STORAGE["f_outer-trace-id"], + name="f_outer", + input={"x": "outer-input"}, + output=None, + error_info={ + "exception_type": "ValueError", + "message": "inner span error message", + "traceback": ANY_STRING(), + }, + ) + + # Verify nested span + verifiers.verify_span( + opik_client=opik_client, + span_id=ID_STORAGE["f_inner-span-id"], + parent_span_id=ID_STORAGE["f_outer-span-id"], + trace_id=ID_STORAGE["f_outer-trace-id"], + name="f_inner", + input={"y": "inner-input"}, + output=None, + error_info={ + "exception_type": "ValueError", + "message": "inner span error message", + "traceback": ANY_STRING(), + }, + ) + + def test_tracked_function__two_traces_and_two_spans__happyflow(opik_client): # Setup project_name = "e2e-tests-batching-messages" diff --git a/sdks/python/tests/e2e/verifiers.py b/sdks/python/tests/e2e/verifiers.py index 7ba678d8f3..0b663da9fd 100644 --- a/sdks/python/tests/e2e/verifiers.py +++ b/sdks/python/tests/e2e/verifiers.py @@ -3,7 +3,7 @@ import json from opik.rest_api import ExperimentPublic -from opik.types import FeedbackScoreDict +from opik.types import FeedbackScoreDict, ErrorInfoDict from opik.api_objects.dataset import dataset_item from opik import Prompt, synchronization @@ -11,6 +11,13 @@ import mock +def _try_get__dict__(instance: Any) -> Optional[Dict[str, Any]]: + if instance is None: + return None + + return instance.__dict__ + + def verify_trace( opik_client: opik.Opik, trace_id: str, @@ -20,7 +27,8 @@ def verify_trace( output: Dict[str, Any] = mock.ANY, # type: ignore tags: List[str] = mock.ANY, # type: ignore feedback_scores: List[FeedbackScoreDict] = mock.ANY, # type: ignore - project_name: Optional[str] = mock.ANY, + project_name: Optional[str] = mock.ANY, # type: ignore + error_info: Optional[ErrorInfoDict] = mock.ANY, # type: ignore ): if not synchronization.until( lambda: (opik_client.get_trace_content(id=trace_id) is not None), @@ -39,6 +47,9 @@ def verify_trace( trace.metadata, metadata ) assert trace.tags == tags, testlib.prepare_difference_report(trace.tags, tags) + assert ( + _try_get__dict__(trace.error_info) == error_info + ), testlib.prepare_difference_report(trace.error_info, error_info) if project_name is not mock.ANY: trace_project = opik_client.get_project(trace.project_id) @@ -88,8 +99,9 @@ def verify_span( type: str = mock.ANY, # type: ignore feedback_scores: List[FeedbackScoreDict] = mock.ANY, # type: ignore project_name: Optional[str] = mock.ANY, - model: Optional[str] = None, - provider: Optional[str] = None, + model: Optional[str] = mock.ANY, # type: ignore + provider: Optional[str] = mock.ANY, # type: ignore + error_info: Optional[ErrorInfoDict] = mock.ANY, # type: ignore ): if not synchronization.until( lambda: (opik_client.get_span_content(id=span_id) is not None), @@ -117,6 +129,9 @@ def verify_span( span.metadata, metadata ) assert span.tags == tags, testlib.prepare_difference_report(span.tags, tags) + assert ( + _try_get__dict__(span.error_info) == error_info + ), testlib.prepare_difference_report(span.error_info, error_info) assert span.model == model assert span.provider == provider