diff --git a/src/aap_eda/core/migrations/0053_alter_ruleset_unique_together_and_more.py b/src/aap_eda/core/migrations/0053_alter_ruleset_unique_together_and_more.py new file mode 100644 index 000000000..1640192ac --- /dev/null +++ b/src/aap_eda/core/migrations/0053_alter_ruleset_unique_together_and_more.py @@ -0,0 +1,26 @@ +# Generated by Django 4.2.16 on 2024-11-22 21:18 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("core", "0052_remove_eventstream_url"), + ] + + operations = [ + migrations.AlterUniqueTogether( + name="ruleset", + unique_together=None, + ), + migrations.RemoveField( + model_name="ruleset", + name="rulebook", + ), + migrations.DeleteModel( + name="Rule", + ), + migrations.DeleteModel( + name="Ruleset", + ), + ] diff --git a/src/aap_eda/core/models/__init__.py b/src/aap_eda/core/models/__init__.py index 30cf95f36..afafee025 100644 --- a/src/aap_eda/core/models/__init__.py +++ b/src/aap_eda/core/models/__init__.py @@ -30,14 +30,7 @@ from .organization import Organization from .project import Project from .queue import ActivationRequestQueue -from .rulebook import ( - AuditAction, - AuditEvent, - AuditRule, - Rule, - Rulebook, - Ruleset, -) +from .rulebook import AuditAction, AuditEvent, AuditRule, Rulebook from .rulebook_process import ( RulebookProcess, RulebookProcessLog, @@ -60,9 +53,7 @@ "JobInstance", "Job", "Project", - "Rule", "Rulebook", - "Ruleset", "User", "AwxToken", "Credential", diff --git a/src/aap_eda/core/models/rulebook.py b/src/aap_eda/core/models/rulebook.py index 3afb40e7f..ef2df7439 100644 --- a/src/aap_eda/core/models/rulebook.py +++ b/src/aap_eda/core/models/rulebook.py @@ -19,8 +19,6 @@ __all__ = ( "Rulebook", - "Ruleset", - "Rule", "AuditRule", ) @@ -62,30 +60,6 @@ def get_rulesets_data(self) -> list[dict]: ) -class Ruleset(models.Model): - class Meta: - db_table = "core_ruleset" - unique_together = ["rulebook_id", "name"] - - name = models.TextField(null=False) - sources = models.JSONField(default=dict) - rulebook = models.ForeignKey( - "Rulebook", on_delete=models.CASCADE, null=True - ) - created_at = models.DateTimeField(auto_now_add=True, null=False) - modified_at = models.DateTimeField(auto_now=True, null=False) - - -class Rule(models.Model): - class Meta: - db_table = "core_rule" - unique_together = ["ruleset", "name"] - - ruleset = models.ForeignKey("Ruleset", on_delete=models.CASCADE, null=True) - name = models.TextField(null=False) - action = models.JSONField(default=dict, null=False) - - class AuditRule(BaseOrgModel): router_basename = "auditrule" diff --git a/src/aap_eda/services/project/imports.py b/src/aap_eda/services/project/imports.py index 716dd647b..d116b0b6b 100644 --- a/src/aap_eda/services/project/imports.py +++ b/src/aap_eda/services/project/imports.py @@ -28,7 +28,6 @@ from aap_eda.core import models from aap_eda.core.types import StrPath from aap_eda.services.project.scm import ScmEmptyError, ScmRepository -from aap_eda.services.rulebook import insert_rulebook_related_data logger = logging.getLogger(__name__) @@ -180,7 +179,6 @@ def _import_rulebook( rulesets=rulebook_info.raw_content, organization=project.organization, ) - insert_rulebook_related_data(rulebook, rulebook_info.content) return rulebook def _sync_rulebook( @@ -196,8 +194,6 @@ def _sync_rulebook( return rulebook.rulesets = rulebook_info.raw_content rulebook.save() - rulebook.ruleset_set.clear() - insert_rulebook_related_data(rulebook, rulebook_info.content) models.Activation.objects.filter(rulebook=rulebook).update( rulebook_rulesets=rulebook.rulesets, git_hash=git_hash, diff --git a/src/aap_eda/services/rulebook.py b/src/aap_eda/services/rulebook.py deleted file mode 100644 index ece8c7fc7..000000000 --- a/src/aap_eda/services/rulebook.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright 2023 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from aap_eda.core import models - - -def expand_ruleset_sources(rulebook_data: dict) -> dict: - # TODO(cutwater): Docstring needed - # TODO(cutwater): Tests needed - expanded_ruleset_sources = {} - if rulebook_data is not None: - for ruleset_data in rulebook_data: - xp_sources = [] - expanded_ruleset_sources[ruleset_data["name"]] = xp_sources - for source in ruleset_data.get("sources") or []: - xp_src = {"name": ""} - for src_key, src_val in source.items(): - if src_key == "name": - xp_src["name"] = src_val - elif src_key == "filters": - xp_src["filters"] = src_val - else: - xp_src["type"] = src_key.split(".")[-1] - xp_src["source"] = src_key - xp_src["config"] = src_val - xp_sources.append(xp_src) - - return expanded_ruleset_sources - - -def insert_rulebook_related_data( - rulebook: models.Rulebook, data: dict -) -> None: - expanded_sources = expand_ruleset_sources(data) - - rule_sets = [ - models.Ruleset( - rulebook=rulebook, - name=data["name"], - sources=expanded_sources.get(data["name"]), - ) - for data in (data or []) - ] - rule_sets = models.Ruleset.objects.bulk_create(rule_sets) - - # Changed to support rules with multiple actions. Will be skipped - # when removing rulebook introspection. - rules = [ - models.Rule( - name=rule.get("name"), - action=rule.get("action") or rule.get("actions", {}), - ruleset=rule_set, - ) - for rule_set, rule_set_data in zip(rule_sets, data) - for rule in rule_set_data["rules"] - ] - models.Rule.objects.bulk_create(rules) diff --git a/tests/integration/api/test_rulebook.py b/tests/integration/api/test_rulebook.py index f4ce68f39..776f6f0d0 100644 --- a/tests/integration/api/test_rulebook.py +++ b/tests/integration/api/test_rulebook.py @@ -518,8 +518,6 @@ def test_delete_project_and_rulebooks( default_project: models.Project, default_activation: models.Activation, default_rulebook: models.Rulebook, - ruleset_1: models.Ruleset, - default_rule: models.Rule, admin_client: APIClient, ): response = admin_client.delete( @@ -531,5 +529,3 @@ def test_delete_project_and_rulebooks( assert activation.rulebook is None assert not models.Project.objects.filter(id=default_project.id).exists() assert not models.Rulebook.objects.filter(id=default_rulebook.id).exists() - assert not models.Ruleset.objects.filter(id=ruleset_1.id).exists() - assert not models.Rule.objects.filter(id=default_rule.id).exists() diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 8d0930e03..0a7af7dd6 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -387,48 +387,6 @@ def source_list() -> List[dict]: ] -@pytest.fixture -def ruleset_1( - default_rulebook: models.Rulebook, source_list: List[dict] -) -> models.Ruleset: - return models.Ruleset.objects.create( - name="ruleset-1", - sources=source_list, - rulebook=default_rulebook, - ) - - -@pytest.fixture -def ruleset_2( - default_rulebook: models.Rulebook, source_list: List[dict] -) -> models.Ruleset: - return models.Ruleset.objects.create( - name="ruleset-2", - sources=source_list, - rulebook=default_rulebook, - ) - - -@pytest.fixture -def ruleset_3( - rulebook_with_job_template: models.Rulebook, source_list: List[dict] -) -> models.Ruleset: - return models.Ruleset.objects.create( - name="ruleset-3", - sources=source_list, - rulebook=rulebook_with_job_template, - ) - - -@pytest.fixture -def default_rule(ruleset_1: models.Ruleset) -> models.Rule: - return models.Rule.objects.create( - name="say hello", - action={"run_playbook": {"name": "ansible.eda.hello"}}, - ruleset=ruleset_1, - ) - - ################################################################# # Activations and Activation Instances ################################################################# diff --git a/tests/integration/services/data/project-01-import.json b/tests/integration/services/data/project-01-import.json deleted file mode 100644 index d472d86a0..000000000 --- a/tests/integration/services/data/project-01-import.json +++ /dev/null @@ -1,42 +0,0 @@ -[ - { - "name": "hello_events.yml", - "rulesets": [ - { - "name": "Hello Events", - "rules": [ - { - "name": "Say Hello", - "action": { - "run_playbook": { - "name": "ansible.eda.hello" - } - } - } - ] - } - ] - }, - { - "name": "kafka/kafka-test-rules.yml", - "rulesets": [ - { - "name": "Demo rules with kafka as source", - "rules": [ - { - "name": "Check defined", - "action": { - "debug": null - } - }, - { - "name": "Shutdown", - "action": { - "shutdown": null - } - } - ] - } - ] - } -] diff --git a/tests/integration/services/data/project-03-import.json b/tests/integration/services/data/project-03-import.json deleted file mode 100644 index f7d8fdb3c..000000000 --- a/tests/integration/services/data/project-03-import.json +++ /dev/null @@ -1,42 +0,0 @@ -[ - { - "name": "hello_events-new.yml", - "rulesets": [ - { - "name": "Hello Events (New)", - "rules": [ - { - "name": "Say Hello (New)", - "action": { - "run_playbook": { - "name": "ansible.eda.hello" - } - } - } - ] - } - ] - }, - { - "name": "kafka/kafka-test-rules.yml", - "rulesets": [ - { - "name": "Demo rules with kafka as source", - "rules": [ - { - "name": "Check defined (updated)", - "action": { - "debug": null - } - }, - { - "name": "Shutdown (updated)", - "action": { - "shutdown": null - } - } - ] - } - ] - } -] diff --git a/tests/integration/services/data/project-05-import.json b/tests/integration/services/data/project-05-import.json deleted file mode 100644 index 45660e816..000000000 --- a/tests/integration/services/data/project-05-import.json +++ /dev/null @@ -1,33 +0,0 @@ -[ - { - "name": "good.yml", - "rulesets": [ - { - "name": "Hello Events", - "rules": [ - { - "name": "Say Hello", - "action": { - "run_playbook": { - "name": "ansible.eda.hello" - } - } - }, - { - "name": "Debug", - "action": [ - { - "print_event": null - }, - { - "debug": { - "msg": "Rule fired correctly" - } - } - ] - } - ] - } - ] - } -] \ No newline at end of file diff --git a/tests/integration/services/test_project_import.py b/tests/integration/services/test_project_import.py index db0eb160e..5f2a7661d 100644 --- a/tests/integration/services/test_project_import.py +++ b/tests/integration/services/test_project_import.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import json import logging import os import re @@ -27,9 +26,6 @@ from aap_eda.services.project.imports import ProjectImportError from aap_eda.services.project.scm import ScmRepository -# TODO(cutwater): The test cases in this test suite share a lot of common code -# and it's ugly. It requires refactoring. - DATA_DIR = Path(__file__).parent / "data" @@ -96,24 +92,31 @@ def projects(default_organization: models.Organization): ) -@pytest.mark.django_db -def test_project_import( - projects: list[models.Project], - storage_save_patch, - service_tempdir_patch, -): +def _mock_git_clone(source: str, git_hash: str) -> mock.Mock: def clone_project(_url, path, *_args, **_kwargs): - src = DATA_DIR / "project-01" + src = DATA_DIR / source shutil.copytree(src, path, symlinks=False) return repo_mock repo_mock = mock.Mock(name="ScmRepository()") - repo_mock.rev_parse.return_value = ( - "adc83b19e793491b1c6ea0fd8b46cd9f32e592fc" - ) + repo_mock.rev_parse.return_value = git_hash + git_mock = mock.Mock(name="ScmRepository", spec=ScmRepository) git_mock.clone.side_effect = clone_project + return git_mock + + +@pytest.mark.django_db +def test_project_import( + projects: list[models.Project], + storage_save_patch, + service_tempdir_patch, +): + git_mock = _mock_git_clone( + "project-01", "adc83b19e793491b1c6ea0fd8b46cd9f32e592fc" + ) + for project in projects: service = ProjectImportService(scm_cls=git_mock) service.import_project(project) @@ -133,15 +136,7 @@ def clone_project(_url, path, *_args, **_kwargs): assert project.git_hash == "adc83b19e793491b1c6ea0fd8b46cd9f32e592fc" assert project.import_state == models.Project.ImportState.COMPLETED - - rulebooks = list(project.rulebook_set.order_by("name")) - assert len(rulebooks) == 2 - - with open(DATA_DIR / "project-01-import.json") as fp: - expected_rulebooks = json.load(fp) - - for rulebook, expected in zip(rulebooks, expected_rulebooks): - assert_rulebook_is_valid(rulebook, expected) + assert project.rulebook_set.count() == 2 @pytest.mark.django_db @@ -150,19 +145,9 @@ def test_project_import_with_new_layout( storage_save_patch, service_tempdir_patch, ): - def clone_project(_url, path, *_args, **_kwargs): - src = DATA_DIR / "project-02" - shutil.copytree(src, path, symlinks=False) - return repo_mock - - repo_mock = mock.Mock(name="ScmRepository()") - repo_mock.rev_parse.return_value = ( - "adc83b19e793491b1c6ea0fd8b46cd9f32e592fc" + git_mock = _mock_git_clone( + "project-02", "adc83b19e793491b1c6ea0fd8b46cd9f32e592fc" ) - - git_mock = mock.Mock(name="ScmRepository", spec=ScmRepository) - git_mock.clone.side_effect = clone_project - service = ProjectImportService(scm_cls=git_mock) service.import_project(project) project.refresh_from_db() @@ -218,19 +203,9 @@ def test_project_import_with_no_rulebooks( storage_save_patch, service_tempdir_patch, ): - def clone_project(_url, path, *_args, **_kwargs): - src = DATA_DIR / "project-06" - shutil.copytree(src, path, symlinks=False) - return repo_mock - - repo_mock = mock.Mock(name="ScmRepository()") - repo_mock.rev_parse.return_value = ( - "adc83b19e793491b1c6ea0fd8b46cd9f32e592fc" + git_mock = _mock_git_clone( + "project-06", "adc83b19e793491b1c6ea0fd8b46cd9f32e592fc" ) - - git_mock = mock.Mock(name="ScmRepository", spec=ScmRepository) - git_mock.clone.side_effect = clone_project - service = ProjectImportService(scm_cls=git_mock) service.import_project(project) project.refresh_from_db() @@ -238,6 +213,7 @@ def clone_project(_url, path, *_args, **_kwargs): assert project.git_hash == "adc83b19e793491b1c6ea0fd8b46cd9f32e592fc" assert project.import_state == models.Project.ImportState.COMPLETED assert project.import_error == "This project contains no rulebooks." + assert project.rulebook_set.count() == 0 @pytest.mark.django_db @@ -246,19 +222,9 @@ def test_project_import_with_vaulted_data( storage_save_patch, service_tempdir_patch, ): - def clone_project(_url, path, *_args, **_kwargs): - src = DATA_DIR / "project-04" - shutil.copytree(src, path, symlinks=False) - return repo_mock - - repo_mock = mock.Mock(name="ScmRepository()") - repo_mock.rev_parse.return_value = ( - "adc83b19e793491b1c6ea0fd8b46cd9f32e592fc" + git_mock = _mock_git_clone( + "project-04", "adc83b19e793491b1c6ea0fd8b46cd9f32e592fc" ) - - git_mock = mock.Mock(name="ScmRepository", spec=ScmRepository) - git_mock.clone.side_effect = clone_project - service = ProjectImportService(scm_cls=git_mock) service.import_project(project) project.refresh_from_db() @@ -267,110 +233,72 @@ def clone_project(_url, path, *_args, **_kwargs): assert project.import_state == models.Project.ImportState.COMPLETED -def _setup_project_sync(default_organization: models.Organization): - def clone_project(_url, path, *_args, **_kwargs): - src = DATA_DIR / "project-02" - shutil.copytree(src, path, symlinks=False) - return repo_mock - - repo_mock = mock.Mock(name="ScmRepository()") - repo_hash = "e5fa44f2b31c1fb553b6021e7360d07d5d91ff5e" - repo_mock.rev_parse.return_value = repo_hash - git_mock = mock.Mock(name="ScmRepository", spec=ScmRepository) - git_mock.clone.side_effect = clone_project - - project = models.Project.objects.create( - name="test-project-01", - url="https://git.example.com/repo.git", - organization=default_organization, +def _setup_project_sync(project: models.Project): + git_mock = _mock_git_clone( + "project-02", "e5fa44f2b31c1fb553b6021e7360d07d5d91ff5e" ) - service = ProjectImportService(scm_cls=git_mock) service.import_project(project) project.refresh_from_db() assert project.git_hash == "e5fa44f2b31c1fb553b6021e7360d07d5d91ff5e" assert project.import_state == models.Project.ImportState.COMPLETED - - rulebooks = list(project.rulebook_set.order_by("name")) - assert len(rulebooks) == 2 + assert project.rulebook_set.count() == 2 return project @pytest.mark.django_db def test_project_sync( + project: models.Project, default_organization: models.Organization, storage_save_patch, service_tempdir_patch, ): # TODO(cutwater): Create activations and verify that rulebook content # is updated - project = _setup_project_sync(default_organization) + _setup_project_sync(project) + rulebooks = list(project.rulebook_set.order_by("name")) + assert len(rulebooks) == 2 + assert rulebooks[0].name == "hello_events.yml" + assert rulebooks[1].name == "kafka/kafka-test-rules.yml" storage_save_patch.reset_mock() - def clone_project(_url, path, *_args, **_kwargs): - src = DATA_DIR / "project-03" - shutil.copytree(src, path, symlinks=False) - return repo_mock - - repo_mock = mock.Mock(name="ScmRepository()") - repo_hash = "7448d8798a4380162d4b56f9b452e2f6f9e24e7a" - repo_mock.rev_parse.return_value = repo_hash - git_mock = mock.Mock(name="ScmRepository", spec=ScmRepository) - git_mock.clone.side_effect = clone_project - + git_mock = _mock_git_clone( + "project-03", "7448d8798a4380162d4b56f9b452e2f6f9e24e7a" + ) service = ProjectImportService(scm_cls=git_mock) service.sync_project(project) project.refresh_from_db() assert project.git_hash == "7448d8798a4380162d4b56f9b452e2f6f9e24e7a" assert project.import_state == models.Project.ImportState.COMPLETED - rulebooks = list(project.rulebook_set.order_by("name")) assert len(rulebooks) == 2 - - with open(DATA_DIR / "project-03-import.json") as fp: - expected_rulebooks = json.load(fp) - - for rulebook, expected in zip(rulebooks, expected_rulebooks): - assert_rulebook_is_valid(rulebook, expected) + assert rulebooks[0].name == "hello_events-new.yml" + assert rulebooks[1].name == "kafka/kafka-test-rules.yml" @pytest.mark.django_db def test_project_sync_same_hash( - default_organization: models.Organization, + project: models.Project, storage_save_patch, service_tempdir_patch, ): - project = _setup_project_sync(default_organization) + _setup_project_sync(project) storage_save_patch.reset_mock() - def clone_project(_url, path, *_args, **_kwargs): - src = DATA_DIR / "project-03" - shutil.copytree(src, path, symlinks=False) - return repo_mock - - repo_mock = mock.Mock(name="ScmRepository()") - repo_hash = "e5fa44f2b31c1fb553b6021e7360d07d5d91ff5e" - repo_mock.rev_parse.return_value = repo_hash - git_mock = mock.Mock(name="ScmRepository", spec=ScmRepository) - git_mock.clone.side_effect = clone_project - + git_mock = _mock_git_clone( + "project-03", "e5fa44f2b31c1fb553b6021e7360d07d5d91ff5e" + ) service = ProjectImportService(scm_cls=git_mock) service.sync_project(project) project.refresh_from_db() assert project.git_hash == "e5fa44f2b31c1fb553b6021e7360d07d5d91ff5e" assert project.import_state == models.Project.ImportState.COMPLETED + assert project.rulebook_set.count() == 2 - rulebooks = list(project.rulebook_set.order_by("name")) - assert len(rulebooks) == 2 - - with open(DATA_DIR / "project-01-import.json") as fp: - expected_rulebooks = json.load(fp) - for rulebook, expected in zip(rulebooks, expected_rulebooks): - assert_rulebook_is_valid(rulebook, expected) storage_save_patch.assert_not_called() @@ -381,19 +309,10 @@ def test_project_import_with_invalid_rulebooks( service_tempdir_patch, caplog, ): - def clone_project(_url, path, *_args, **_kwargs): - src = DATA_DIR / "project-05" - shutil.copytree(src, path, symlinks=False) - return repo_mock - - repo_mock = mock.Mock(name="ScmRepository()") - repo_mock.rev_parse.return_value = ( - "adc83b19e793491b1c6ea0fd8b46cd9f32e592fc" + git_mock = _mock_git_clone( + "project-05", "adc83b19e793491b1c6ea0fd8b46cd9f32e592fc" ) - git_mock = mock.Mock(name="ScmRepository", spec=ScmRepository) - git_mock.clone.side_effect = clone_project - logger = logging.getLogger("aap_eda") propagate = logger.propagate logger.propagate = True @@ -408,36 +327,4 @@ def clone_project(_url, path, *_args, **_kwargs): assert project.git_hash == "adc83b19e793491b1c6ea0fd8b46cd9f32e592fc" assert project.import_state == models.Project.ImportState.COMPLETED assert caplog.text.count("WARNING") == 10 - - rulebooks = list(project.rulebook_set.order_by("name")) - assert len(rulebooks) == 1 - - with open(DATA_DIR / "project-05-import.json") as fp: - expected_rulebooks = json.load(fp) - for rulebook, expected in zip(rulebooks, expected_rulebooks): - assert_rulebook_is_valid(rulebook, expected) - - -def assert_rulebook_is_valid(rulebook: models.Rulebook, expected: dict): - assert rulebook.name == expected["name"] - - rulesets = list(rulebook.ruleset_set.order_by("id")) - assert len(rulesets) == len(expected["rulesets"]) - - for ruleset, expected_rulesets in zip(rulesets, expected["rulesets"]): - assert_ruleset_is_valid(ruleset, expected_rulesets) - - -def assert_ruleset_is_valid(ruleset: models.Ruleset, expected: dict): - assert ruleset.name == expected["name"] - - rules = list(ruleset.rule_set.order_by("id")) - assert len(rules) == len(expected["rules"]) - - for rule, expected_rules in zip(rules, expected["rules"]): - assert_rule_is_valid(rule, expected_rules) - - -def assert_rule_is_valid(rule: models.Rule, expected: dict): - assert rule.name == expected["name"] - assert rule.action == expected["action"] + assert project.rulebook_set.count() == 1 diff --git a/tests/integration/wsapi/test_consumer.py b/tests/integration/wsapi/test_consumer.py index f89c4800a..9eaed1643 100644 --- a/tests/integration/wsapi/test_consumer.py +++ b/tests/integration/wsapi/test_consumer.py @@ -907,25 +907,6 @@ def _prepare_db_data( organization=default_organization, ) - ruleset, _ = models.Ruleset.objects.get_or_create( - name="ruleset", - sources=[ - { - "name": "", - "type": "range", - "config": {"limit": 5}, - "source": "ansible.eda.range", - } - ], - rulebook=rulebook, - ) - - rule, _ = models.Rule.objects.get_or_create( - name="rule", - action={"run_playbook": {"name": "ansible.eda.hello"}}, - ruleset=ruleset, - ) - return rulebook_process.id