OPIK-364 feedback definitions crud tests

.github/workflows/end2end_suites.yml

@@ -16,6 +16,7 @@ on:
                 - datasets
                 - experiments
                 - prompts
+                - feedback_definitions
     pull_request:
       paths:
         - 'sdks/code_generation/fern/openapi/openapi.yaml'
@@ -91,6 +92,8 @@ jobs:
                     pytest -s tests/Experiments/test_experiments_crud_operations.py --browser chromium --base-url http://localhost:5173 --setup-show
                 elif [ "$SUITE" == "prompts" ]; then
                     pytest -s tests/Prompts/test_prompts_crud_operations.py --browser chromium --base-url http://localhost:5173 --setup-show
+                elif [ "$SUITE" == "feedback_definitions" ]; then
+                    pytest -s tests/FeedbackDefinitions/test_feedback_definitions_crud.py --browser chromium --base-url http://localhost:5173 --setup-show
                 elif [ "$SUITE" == "sanity" ]; then
                     pytest -s tests/application_sanity/test_sanity.py --browser chromium --base-url http://localhost:5173 --setup-show
                 elif [ "$SUITE" == "all_features" ]; then

tests_end_to_end/page_objects/FeedbackDefinitionsPage.py

@@ -0,0 +1,140 @@
+from playwright.sync_api import Page, expect
+from typing import Literal, Optional
+
+
+class FeedbackDefinitionsPage:
+    def __init__(self, page: Page):
+        self.page = page
+        self.url = "/default/configuration?tab=feedback-definitions"
+        self.search_bar = self.page.get_by_test_id("search-input")
+
+    def go_to_page(self):
+        self.page.goto(self.url)
+
+    def search_feedback_by_name(self, feedback_name: str):
+        self.search_bar.click()
+        self.search_bar.fill(feedback_name)
+
+    def fill_categorical_values(self, categories):
+        if not categories:
+            category1_name = self.page.get_by_placeholder("Name").nth(2)
+            category1_val = self.page.get_by_placeholder("0.0").first
+            category2_name = self.page.get_by_placeholder("Name").nth(3)
+            category2_val = self.page.get_by_placeholder("0.0").nth(1)
+
+            category1_name.click()
+            category1_name.fill("a")
+            category1_val.click()
+            category1_val.fill("1")
+
+            category2_name.click()
+            category2_name.fill("b")
+            category2_val.click()
+            category2_val.fill("2")
+
+        else:
+            if len(categories.keys()) == 1:
+                raise ValueError(
+                    "At least 2 categories are required for Categorical feedback definition"
+                )
+            for i, key in enumerate(categories.keys()):
+                self.page.get_by_placeholder("Name").nth(i + 2).click()
+                self.page.get_by_placeholder("Name").nth(i + 2).fill(key)
+                self.page.get_by_placeholder("0.0").nth(i).click()
+                self.page.get_by_placeholder("0.0").nth(i).fill(str(categories[key]))
+                self.page.get_by_role("button", name="Add category").click()
+
+    def fill_numerical_values(self, min, max):
+        min_box = self.page.get_by_placeholder("Min")
+        max_box = self.page.get_by_placeholder("Max")
+
+        both_values_provided = min is not None and max is not None
+
+        min_box.click()
+        val = min if both_values_provided else 0
+        min_box.fill(str(val))
+
+        max_box.click()
+        val = max if both_values_provided else 1
+        max_box.fill(str(val))
+
+    def create_new_feedback(
+        self,
+        feedback_name: str,
+        feedback_type: Literal["categorical", "numerical"],
+        categories: Optional[dict] = None,
+        min: Optional[int] = None,
+        max: Optional[int] = None,
+    ):
+        self.page.get_by_role(
+            "button", name="Create new feedback definition"
+        ).first.click()
+        self.page.get_by_placeholder("Feedback definition name").fill(feedback_name)
+        self.page.get_by_role("combobox").click()
+        self.page.get_by_label(feedback_type.capitalize()).click()
+        if feedback_type == "categorical":
+            self.fill_categorical_values(categories=categories)
+        else:
+            self.fill_numerical_values(min=min, max=max)
+        self.page.get_by_role("button", name="Create feedback definition").click()
+
+    def check_feedback_exists_by_name(self, feedback_name: str):
+        self.search_feedback_by_name(feedback_name=feedback_name)
+        expect(self.page.get_by_text(feedback_name).first).to_be_visible()
+        self.search_bar.fill("")
+
+    def check_feedback_not_exists_by_name(self, feedback_name: str):
+        self.search_feedback_by_name(feedback_name=feedback_name)
+        expect(self.page.get_by_text(feedback_name).first).not_to_be_visible()
+        self.search_bar.fill("")
+
+    def delete_feedback_by_name(self, feedback_name: str):
+        self.search_feedback_by_name(feedback_name=feedback_name)
+        expect(self.page.get_by_role("row", name=feedback_name).first).to_be_visible()
+        self.page.get_by_role("row", name=feedback_name).first.get_by_role(
+            "button", name="Actions menu"
+        ).click()
+        self.page.get_by_role("menuitem", name="Delete").click()
+        self.page.get_by_role("button", name="Delete feedback definition").click()
+        self.search_bar.fill("")
+
+    def edit_feedback_by_name(
+        self,
+        feedback_name: str,
+        new_name: str = "",
+        feedback_type: Optional[Literal["categorical", "numerical"]] = None,
+        categories: Optional[dict] = None,
+        min: Optional[int] = None,
+        max: Optional[int] = None,
+    ):
+        self.search_feedback_by_name(feedback_name=feedback_name)
+        self.page.get_by_role("row", name=feedback_name).first.get_by_role(
+            "button", name="Actions menu"
+        ).click()
+        self.page.get_by_role("menuitem", name="Edit").click()
+
+        if new_name:
+            self.page.get_by_placeholder("Feedback definition name").fill(new_name)
+        ftype = feedback_type or self.page.get_by_role("combobox").inner_text()
+        if ftype.lower() == "categorical":
+            # currently only supporting resetting the category values entirely, will add entering new categories on top of the old ones later if needed
+            self.fill_categorical_values(categories=categories)
+        else:
+            self.fill_numerical_values(min=min, max=max)
+
+        self.page.get_by_role("button", name="Update feedback definition").click()
+        self.search_bar.fill("")
+
+    def get_type_of_feedback_by_name(self, feedback_name: str):
+        self.search_feedback_by_name(feedback_name=feedback_name)
+        self.page.wait_for_timeout(500)
+        return (
+            self.page.get_by_role("row").nth(1).get_by_role("cell").nth(2).inner_text()
+        )
+
+    def get_values_of_feedback_by_name(self, feedback_name: str):
+        self.search_feedback_by_name(feedback_name=feedback_name)
+        self.page.wait_for_timeout(500)
+        return (
+            self.page.get_by_role("row").nth(1).get_by_role("cell").nth(3).inner_text()
+        )

tests_end_to_end/tests/FeedbackDefinitions/test_feedback_definitions_crud.py

@@ -0,0 +1,110 @@
+import re
+from playwright.sync_api import Page
+from page_objects.FeedbackDefinitionsPage import FeedbackDefinitionsPage
+from collections import Counter
+
+
+class TestFeedbacksCrud:
+    def test_feedback_definition_visibility(
+        self,
+        page: Page,
+        create_feedback_definition_categorical_ui,
+        create_feedback_definition_numerical_ui,
+    ):
+        """
+        Creates a categorical and numerical feedback definition and checks they are properly displayed in the UI
+        1. Create 2 feedback definitions (categorical and numerical)
+        2. Check the feedback definitions appear in the table
+        """
+        feedbacks_page = FeedbackDefinitionsPage(page)
+        feedbacks_page.go_to_page()
+        feedbacks_page.check_feedback_exists_by_name(
+            create_feedback_definition_categorical_ui["name"]
+        )
+        feedbacks_page.check_feedback_exists_by_name(
+            create_feedback_definition_numerical_ui["name"]
+        )
+
+    def test_feedback_definition_edit(
+        self,
+        page: Page,
+        create_feedback_definition_categorical_ui,
+        create_feedback_definition_numerical_ui,
+    ):
+        """
+        Tests that updating the data of feedback definition correctly displays in the UI
+        1. Create 2 feedback definitions (categorical and numerical)
+        2. Update the name of the 2 feedbacks
+        3. Update the values of the 2 feedbacks (change the categories and the min-max values, respectively)
+        4. Check that the new names are properly displayed in the table
+        5. Check that the new values are properly displayed in the table
+        """
+        feedbacks_page = FeedbackDefinitionsPage(page)
+        feedbacks_page.go_to_page()
+
+        fd_cat_name = create_feedback_definition_categorical_ui["name"]
+        fd_num_name = create_feedback_definition_numerical_ui["name"]
+
+        new_categories = {"test1": 1, "test2": 2, "test3": 3}
+        new_min = 5
+        new_max = 10
+        cat_new_name = "updated_name_categorical"
+        num_new_name = "updated_name_numerical"
+
+        feedbacks_page.edit_feedback_by_name(
+            feedback_name=fd_cat_name, new_name=cat_new_name, categories=new_categories
+        )
+        create_feedback_definition_categorical_ui["name"] = cat_new_name
+
+        feedbacks_page.edit_feedback_by_name(
+            feedback_name=fd_num_name, new_name=num_new_name, min=new_min, max=new_max
+        )
+        create_feedback_definition_numerical_ui["name"] = num_new_name
+
+        feedbacks_page.check_feedback_exists_by_name(cat_new_name)
+        feedbacks_page.check_feedback_exists_by_name(num_new_name)
+
+        assert (
+            feedbacks_page.get_type_of_feedback_by_name(cat_new_name) == "Categorical"
+        )
+        assert feedbacks_page.get_type_of_feedback_by_name(num_new_name) == "Numerical"
+
+        categories_ui_values = feedbacks_page.get_values_of_feedback_by_name(
+            cat_new_name
+        )
+        categories = re.findall(r"\b\w+\b", categories_ui_values)
+        assert Counter(categories) == Counter(new_categories.keys())
+
+        numerical_ui_values = feedbacks_page.get_values_of_feedback_by_name(
+            num_new_name
+        )
+        match = re.search(r"Min: (\d+), Max: (\d+)", numerical_ui_values)
+        assert match is not None, "Improper formatting of min-max values"
+        min_value = match.group(1)
+        max_value = match.group(2)
+        assert int(min_value) == new_min
+        assert int(max_value) == new_max
+
+    def test_feedback_definition_deletion(
+        self,
+        page: Page,
+        create_feedback_definition_categorical_ui,
+        create_feedback_definition_numerical_ui,
+    ):
+        """
+        Checks that deleting feedback definitions properly removes them from the table
+        1. Create 2 feedback definitions (categorical and numerical)
+        2. Delete them
+        3. Check that they no longer appear in the table
+        """
+        feedbacks_page = FeedbackDefinitionsPage(page)
+        feedbacks_page.go_to_page()
+
+        fd_cat_name = create_feedback_definition_categorical_ui["name"]
+        fd_num_name = create_feedback_definition_numerical_ui["name"]
+
+        feedbacks_page.delete_feedback_by_name(feedback_name=fd_cat_name)
+        feedbacks_page.delete_feedback_by_name(feedback_name=fd_num_name)
+
+        feedbacks_page.check_feedback_not_exists_by_name(feedback_name=fd_cat_name)
+        feedbacks_page.check_feedback_not_exists_by_name(feedback_name=fd_num_name)

tests_end_to_end/tests/conftest.py

@@ -7,6 +7,7 @@
 from page_objects.DatasetsPage import DatasetsPage
 from page_objects.ExperimentsPage import ExperimentsPage
 from page_objects.PromptLibraryPage import PromptLibraryPage
+from page_objects.FeedbackDefinitionsPage import FeedbackDefinitionsPage
 from tests.sdk_helpers import (
     create_project_sdk,
     delete_project_by_name_sdk,
@@ -205,3 +206,37 @@ def create_10_test_traces(page: Page, client, create_delete_project_sdk):
         )
     wait_for_number_of_traces_to_be_visible(project_name=proj_name, number_of_traces=10)
     yield
+
+
+@pytest.fixture
+def create_feedback_definition_categorical_ui(client: opik.Opik, page: Page):
+    feedbacks_page = FeedbackDefinitionsPage(page)
+    feedbacks_page.go_to_page()
+    feedbacks_page.create_new_feedback(
+        feedback_name="feedback_c_test", feedback_type="categorical"
+    )
+
+    # passing it to the test as a mutable type to cover name change edits
+    data = {"name": "feedback_c_test"}
+    yield data
+    try:
+        feedbacks_page.check_feedback_not_exists_by_name(feedback_name=data["name"])
+    except AssertionError as _:
+        feedbacks_page.delete_feedback_by_name(data["name"])
+
+
+@pytest.fixture
+def create_feedback_definition_numerical_ui(client: opik.Opik, page: Page):
+    feedbacks_page = FeedbackDefinitionsPage(page)
+    feedbacks_page.go_to_page()
+    feedbacks_page.create_new_feedback(
+        feedback_name="feedback_n_test", feedback_type="numerical"
+    )
+
+    # passing it to the test as a mutable type to cover name change edits
+    data = {"name": "feedback_n_test"}
+    yield data
+    try:
+        feedbacks_page.check_feedback_not_exists_by_name(feedback_name=data["name"])
+    except AssertionError as _:
+        feedbacks_page.delete_feedback_by_name(data["name"])