diff --git a/.github/workflows/end2end_suites.yml b/.github/workflows/end2end_suites.yml index 5c81931666..41855a7884 100644 --- a/.github/workflows/end2end_suites.yml +++ b/.github/workflows/end2end_suites.yml @@ -91,6 +91,8 @@ jobs: pytest -s tests/Experiments/test_experiments_crud_operations.py --browser chromium --base-url http://localhost:5173 --setup-show elif [ "$SUITE" == "prompts" ]; then pytest -s tests/Prompts/test_prompts_crud_operations.py --browser chromium --base-url http://localhost:5173 --setup-show + elif [ "$SUITE" == "feedbacks" ]; then + pytest -s tests/Feedbacks/test_feedbacks_crud.py --browser chromium --base-url http://localhost:5173 --setup-show elif [ "$SUITE" == "sanity" ]; then pytest -s tests/application_sanity/test_sanity.py --browser chromium --base-url http://localhost:5173 --setup-show elif [ "$SUITE" == "all_features" ]; then diff --git a/tests_end_to_end/page_objects/FeedbackDefinitionsPage.py b/tests_end_to_end/page_objects/FeedbackDefinitionsPage.py new file mode 100644 index 0000000000..31f7c54402 --- /dev/null +++ b/tests_end_to_end/page_objects/FeedbackDefinitionsPage.py @@ -0,0 +1,116 @@ +from playwright.sync_api import Page, expect +from typing import Literal + + +class FeedbackDefinitionsPage: + def __init__(self, page: Page): + self.page = page + self.url = "/default/configuration?tab=feedback-definitions" + self.search_bar = self.page.get_by_test_id("search-input") + + def go_to_page(self): + self.page.goto(self.url) + + def search_feedback_by_name(self, feedback_name: str): + self.search_bar.click() + self.search_bar.fill(feedback_name) + + def fill_categorical_values(self, categories): + if not categories: + category1_name = self.page.get_by_placeholder("Name").nth(2) + category1_val = self.page.get_by_placeholder("0.0").first + category2_name = self.page.get_by_placeholder("Name").nth(3) + category2_val = self.page.get_by_placeholder("0.0").nth(1) + + category1_name.click() + category1_name.fill("a") + category1_val.click() + category1_val.fill("1") + + category2_name.click() + category2_name.fill("b") + category2_val.click() + category2_val.fill("2") + + else: + if len(categories.keys()) == 1: + raise ValueError("At least 2 categories are required for Categorical feedback definition") + for i, key in enumerate(categories.keys()): + self.page.get_by_placeholder("Name").nth(i+2).click() + self.page.get_by_placeholder("Name").nth(i+2).fill(key) + self.page.get_by_placeholder("0.0").nth(i).click() + self.page.get_by_placeholder("0.0").nth(i).fill(str(categories[key])) + self.page.get_by_role("button", name="Add category").click() + + def fill_numerical_values(self, min, max): + min_box = self.page.get_by_placeholder("Min") + max_box = self.page.get_by_placeholder("Max") + + both_values_provided = (min is not None and max is not None) + + min_box.click() + val = min if both_values_provided else 0 + min_box.fill(str(val)) + + max_box.click() + val = max if both_values_provided else 1 + max_box.fill(str(val)) + + def create_new_feedback(self, feedback_name: str, feedback_type: Literal["categorical", "numerical"], categories: dict=None, min: int=None, max: int=None): + self.page.get_by_role("button", name="Create new feedback definition").click() + self.page.get_by_placeholder("Feedback definition name").fill(feedback_name) + self.page.get_by_role("combobox").click() + self.page.get_by_label(feedback_type.capitalize()).click() + if feedback_type == "categorical": + self.fill_categorical_values(categories=categories) + else: + self.fill_numerical_values(min=min, max=max) + self.page.get_by_role("button", name="Create feedback definition").click() + + def check_feedback_exists_by_name(self, feedback_name: str): + self.search_feedback_by_name(feedback_name=feedback_name) + expect(self.page.get_by_text(feedback_name).first).to_be_visible() + self.search_bar.fill("") + + def check_feedback_not_exists_by_name(self, feedback_name: str): + self.search_feedback_by_name(feedback_name=feedback_name) + expect(self.page.get_by_text(feedback_name).first).not_to_be_visible() + self.search_bar.fill("") + + def delete_feedback_by_name(self, feedback_name: str): + self.search_feedback_by_name(feedback_name=feedback_name) + expect(self.page.get_by_role("row", name=feedback_name).first).to_be_visible() + self.page.get_by_role("row", name=feedback_name).first.get_by_role( + "button", name="Actions menu" + ).click() + self.page.get_by_role("menuitem", name="Delete").click() + self.page.get_by_role("button", name="Delete feedback definition").click() + self.search_bar.fill("") + + def edit_feedback_by_name(self, feedback_name: str, new_name: str=None, feedback_type: Literal["categorical", "numerical"]=None, categories: dict=None, min: int=None, max: int=None): + self.search_feedback_by_name(feedback_name=feedback_name) + self.page.get_by_role("row", name=feedback_name).first.get_by_role( + "button", name="Actions menu").click() + self.page.get_by_role("menuitem", name="Edit").click() + + if new_name: + self.page.get_by_placeholder("Feedback definition name").fill(new_name) + ftype = feedback_type or self.page.get_by_role("combobox").inner_text() + if ftype.lower() == "categorical": + # currently only supporting resetting the category values entirely, will add entering new categories on top of the old ones later if needed + self.fill_categorical_values(categories=categories) + else: + self.fill_numerical_values(min=min, max=max) + + self.page.get_by_role("button", name="Update feedback definition").click() + self.search_bar.fill("") + + def get_type_of_feedback_by_name(self, feedback_name: str): + self.search_feedback_by_name(feedback_name=feedback_name) + self.page.wait_for_timeout(500) + return self.page.get_by_role("row").nth(1).get_by_role("cell").nth(2).inner_text() + + def get_values_of_feedback_by_name(self, feedback_name: str): + self.search_feedback_by_name(feedback_name=feedback_name) + self.page.wait_for_timeout(500) + return self.page.get_by_role("row").nth(1).get_by_role("cell").nth(3).inner_text() \ No newline at end of file diff --git a/tests_end_to_end/tests/Feedbacks/test_feedbacks_crud.py b/tests_end_to_end/tests/Feedbacks/test_feedbacks_crud.py new file mode 100644 index 0000000000..0a6a4c75ca --- /dev/null +++ b/tests_end_to_end/tests/Feedbacks/test_feedbacks_crud.py @@ -0,0 +1,106 @@ +import pytest +import re +from playwright.sync_api import Page +from page_objects.FeedbackDefinitionsPage import FeedbackDefinitionsPage +from collections import Counter + + +class TestFeedbacksCrud: + def test_feedback_visibility(self, page: Page, create_feedback_definition_categorical_ui, create_feedback_definition_numerical_ui): + """ + Creates a categorical and numerical feedback definition and checks they are properly displayed in the UI + 1. Create 2 feedback definitions (categorical and numerical) + 2. Check the feedback definitions appear in the table + """ + feedbacks_page = FeedbackDefinitionsPage(page) + feedbacks_page.go_to_page() + feedbacks_page.check_feedback_exists_by_name( + create_feedback_definition_categorical_ui["name"] + ) + feedbacks_page.check_feedback_exists_by_name( + create_feedback_definition_numerical_ui["name"] + ) + + def test_feedback_edit(self, page: Page, create_feedback_definition_categorical_ui, create_feedback_definition_numerical_ui): + """ + Tests that updating the data of feedback definition correctly displays in the UI + 1. Create 2 feedback definitions (categorical and numerical) + 2. Update the name of the 2 feedbacks + 3. Update the values of the 2 feedbacks (change the categories and the min-max values, respectively) + 4. Check that the new names are properly displayed in the table + 5. Check that the new values are properly displayed in the table + """ + feedbacks_page = FeedbackDefinitionsPage(page) + feedbacks_page.go_to_page() + + fd_cat_name = create_feedback_definition_categorical_ui["name"] + fd_num_name = create_feedback_definition_numerical_ui["name"] + + new_categories = { + "test1": 1, + "test2": 2, + "test3": 3 + } + new_min = 5 + new_max = 10 + cat_new_name = "updated_name_categorical" + num_new_name = "updated_name_numerical" + + feedbacks_page.edit_feedback_by_name( + feedback_name=fd_cat_name, + new_name=cat_new_name, + categories=new_categories + ) + create_feedback_definition_categorical_ui["name"] = cat_new_name + + feedbacks_page.edit_feedback_by_name( + feedback_name=fd_num_name, + new_name=num_new_name, + min=new_min, + max=new_max + ) + create_feedback_definition_numerical_ui["name"] = num_new_name + + feedbacks_page.check_feedback_exists_by_name(cat_new_name) + feedbacks_page.check_feedback_exists_by_name(num_new_name) + + assert feedbacks_page.get_type_of_feedback_by_name(cat_new_name) == "Categorical" + assert feedbacks_page.get_type_of_feedback_by_name(num_new_name) == "Numerical" + + categories_ui_values = feedbacks_page.get_values_of_feedback_by_name(cat_new_name) + categories = re.findall(r"\b\w+\b", categories_ui_values) + assert Counter(categories) == Counter(new_categories.keys()) + + numerical_ui_values = feedbacks_page.get_values_of_feedback_by_name(num_new_name) + match = re.search(r"Min: (\d+), Max: (\d+)", numerical_ui_values) + min_value = match.group(1) + max_value = match.group(2) + assert int(min_value) == new_min + assert int(max_value) == new_max + + def test_feedback_definition_deletion(self, page: Page, create_feedback_definition_categorical_ui, create_feedback_definition_numerical_ui): + """ + Checks that deleting feedback definitions properly removes them from the table + 1. Create 2 feedback definitions (categorical and numerical) + 2. Delete them + 3. Check that they no longer appear in the table + """ + feedbacks_page = FeedbackDefinitionsPage(page) + feedbacks_page.go_to_page() + + fd_cat_name = create_feedback_definition_categorical_ui["name"] + fd_num_name = create_feedback_definition_numerical_ui["name"] + + feedbacks_page.delete_feedback_by_name( + feedback_name=fd_cat_name + ) + feedbacks_page.delete_feedback_by_name( + feedback_name=fd_num_name + ) + + feedbacks_page.check_feedback_not_exists_by_name( + feedback_name=fd_cat_name + ) + feedbacks_page.check_feedback_not_exists_by_name( + feedback_name=fd_num_name + ) diff --git a/tests_end_to_end/tests/conftest.py b/tests_end_to_end/tests/conftest.py index 26543b2eab..9b71d901d5 100644 --- a/tests_end_to_end/tests/conftest.py +++ b/tests_end_to_end/tests/conftest.py @@ -7,6 +7,7 @@ from page_objects.DatasetsPage import DatasetsPage from page_objects.ExperimentsPage import ExperimentsPage from page_objects.PromptLibraryPage import PromptLibraryPage +from page_objects.FeedbackDefinitionsPage import FeedbackDefinitionsPage from tests.sdk_helpers import ( create_project_sdk, delete_project_by_name_sdk, @@ -205,3 +206,37 @@ def create_10_test_traces(page: Page, client, create_delete_project_sdk): ) wait_for_number_of_traces_to_be_visible(project_name=proj_name, number_of_traces=10) yield + + +@pytest.fixture +def create_feedback_definition_categorical_ui(client: opik.Opik, page: Page): + feedbacks_page = FeedbackDefinitionsPage(page) + feedbacks_page.go_to_page() + feedbacks_page.create_new_feedback( + feedback_name="feedback_c_test", feedback_type="categorical" + ) + + # passing it to the test as a mutable type to cover name change edits + data = {"name": "feedback_c_test"} + yield data + try: + feedbacks_page.check_feedback_not_exists_by_name(feedback_name=data["name"]) + except AssertionError as _: + feedbacks_page.delete_feedback_by_name(data["name"]) + + +@pytest.fixture +def create_feedback_definition_numerical_ui(client: opik.Opik, page: Page): + feedbacks_page = FeedbackDefinitionsPage(page) + feedbacks_page.go_to_page() + feedbacks_page.create_new_feedback( + feedback_name="feedback_n_test", feedback_type="numerical" + ) + + # passing it to the test as a mutable type to cover name change edits + data = {"name": "feedback_n_test"} + yield data + try: + feedbacks_page.check_feedback_not_exists_by_name(feedback_name=data["name"]) + except AssertionError as _: + feedbacks_page.delete_feedback_by_name(data["name"])