Mkerge branch 'main' of github.com:codecov/worker into 3210-batch-upl…

…oads-creation

VERSION

@@ -1 +1 @@
-25.1.10
+25.1.16

database/enums.py

@@ -68,7 +68,3 @@ class FlakeSymptomType(Enum):
     FAILED_IN_DEFAULT_BRANCH = "failed_in_default_branch"
     CONSECUTIVE_DIFF_OUTCOMES = "consecutive_diff_outcomes"
     UNRELATED_MATCHING_FAILURES = "unrelated_matching_failures"
-
-
-class TestResultsProcessingError(Enum):
-    NO_SUCCESS = "no_success"

database/events.py

@@ -6,12 +6,19 @@
 from sqlalchemy import event, inspect
 
 from database.models.core import Repository
+from helpers.environment import is_enterprise
 
 _pubsub_publisher = None
 
 log = logging.getLogger(__name__)
 
 
+def _is_shelter_enabled():
+    return get_config(
+        "setup", "shelter", "enabled", default=False if is_enterprise() else True
+    )
+
+
 def _get_pubsub_publisher():
     global _pubsub_publisher
     if not _pubsub_publisher:
@@ -45,12 +52,22 @@ def _sync_repo(repository: Repository):
 
 @event.listens_for(Repository, "after_insert")
 def after_insert_repo(mapper, connection, target: Repository):
+    if not _is_shelter_enabled():
+        log.debug("Shelter is not enabled, skipping after_insert signal")
+        return
+
+    # Send to shelter service
     log.info("After insert signal", extra=dict(repoid=target.repoid))
     _sync_repo(target)
 
 
 @event.listens_for(Repository, "after_update")
 def after_update_repo(mapper, connection, target: Repository):
+    if not _is_shelter_enabled():
+        log.debug("Shelter is not enabled, skipping after_update signal")
+        return
+
+    # Send to shelter service
     state = inspect(target)
 
     for attr in state.attrs:

database/models/core.py

@@ -3,6 +3,7 @@
 import uuid
 from datetime import datetime
 from functools import cached_property
+from typing import Optional
 
 from shared.plan.constants import PlanName
 from sqlalchemy import Column, ForeignKey, Index, UniqueConstraint, types
@@ -105,6 +106,7 @@ class Owner(CodecovBaseModel):
     avatar_url = Column(types.Text, server_default=FetchedValue())
     updatestamp = Column(types.DateTime, server_default=FetchedValue())
     parent_service_id = Column(types.Text, server_default=FetchedValue())
+    root_parent_service_id = Column(types.Text, nullable=True)
     plan_provider = Column(types.Text, server_default=FetchedValue())
     trial_start_date = Column(types.DateTime, server_default=FetchedValue())
     trial_end_date = Column(types.DateTime, server_default=FetchedValue())
@@ -153,10 +155,43 @@ class Owner(CodecovBaseModel):
     )
 
     @property
-    def slug(self):
+    def slug(self: "Owner") -> str:
         return self.username
 
-    def __repr__(self):
+    @property
+    def root_organization(self: "Owner") -> Optional["Owner"]:
+        """
+        Find the root organization of Gitlab OwnerOrg, by using the root_parent_service_id
+        if it exists, otherwise iterating through the parents and cache it in root_parent_service_id
+        """
+        db_session = self.get_db_session()
+        if self.root_parent_service_id:
+            return self._get_owner_by_service_id(
+                db_session, self.root_parent_service_id
+            )
+
+        root = None
+        if self.service == "gitlab" and self.parent_service_id:
+            root = self
+            while root.parent_service_id is not None:
+                root = self._get_owner_by_service_id(db_session, root.parent_service_id)
+            self.root_parent_service_id = root.service_id
+            db_session.commit()
+        return root
+
+    def _get_owner_by_service_id(
+        self: "Owner", db_session: Session, service_id: str
+    ) -> "Owner":
+        """
+        Helper method to fetch an Owner by service_id.
+        """
+        return (
+            db_session.query(Owner)
+            .filter_by(service_id=service_id, service=self.service)
+            .one()
+        )
+
+    def __repr__(self: "Owner") -> str:
         return f"Owner<{self.ownerid}@service<{self.service}>>"
 
 

database/models/reports.py

@@ -331,6 +331,9 @@ class TestResultReportTotals(CodecovBaseModel, MixinBaseClass):
     passed = Column(types.Integer)
     skipped = Column(types.Integer)
     failed = Column(types.Integer)
+
+    # this field is no longer used in the new ta_finisher task
+    # TODO: thus, it will be removed in the future
     error = Column(types.String(100), nullable=True)
 
 

database/tests/factories/core.py

@@ -4,6 +4,7 @@
 
 import factory
 from factory import Factory
+from shared.plan.constants import PlanName
 
 from database import enums, models
 from services.encryption import encryptor
@@ -49,6 +50,7 @@ class Meta:
     trial_status = enums.TrialStatus.NOT_STARTED.value
     trial_fired_by = None
     upload_token_required_for_public_repos = False
+    plan = PlanName.BASIC_PLAN_NAME.value
 
     oauth_token = factory.LazyAttribute(
         lambda o: encrypt_oauth_token(o.unencrypted_oauth_token)
@@ -296,3 +298,12 @@ class Meta:
 
     key = ""
     value = ""
+
+
+class UploadErrorFactory(Factory):
+    class Meta:
+        model = models.UploadError
+
+    report_upload = factory.SubFactory(UploadFactory)
+    error_code = "error"
+    error_params = {"error_message": "error message"}

database/tests/factories/reports.py

@@ -2,8 +2,18 @@
 
 import factory
 
-from database.models.reports import CompareFlag, Flake, RepositoryFlag, Test
-from database.tests.factories.core import CompareCommitFactory, RepositoryFactory
+from database.models.reports import (
+    CompareFlag,
+    Flake,
+    RepositoryFlag,
+    Test,
+    TestResultReportTotals,
+)
+from database.tests.factories.core import (
+    CompareCommitFactory,
+    ReportFactory,
+    RepositoryFactory,
+)
 
 
 class RepositoryFlagFactory(factory.Factory):
@@ -47,3 +57,13 @@ class Meta:
 
     start_date = dt.datetime.now()
     end_date = None
+
+
+class TestResultReportTotalsFactory(factory.Factory):
+    class Meta:
+        model = TestResultReportTotals
+
+    report = factory.SubFactory(ReportFactory)
+    passed = 0
+    skipped = 0
+    failed = 0

database/tests/unit/test_events.py

@@ -14,6 +14,7 @@ def test_shelter_repo_sync(dbsession, mock_configuration, mocker):
                 "shelter": {
                     "pubsub_project_id": "test-project-id",
                     "sync_repo_topic_id": "test-topic-id",
+                    "enabled": True,
                 }
             }
         }
@@ -53,3 +54,30 @@ def test_shelter_repo_sync(dbsession, mock_configuration, mocker):
         "projects/test-project-id/topics/test-topic-id",
         b'{"type": "repo", "sync": "one", "id": 91728376}',
     )
+
+
+def test_repo_sync_when_shelter_disabled(dbsession, mock_configuration, mocker):
+    # this prevents the pubsub SDK from trying to load credentials
+    os.environ["PUBSUB_EMULATOR_HOST"] = "localhost"
+
+    mock_configuration.set_params(
+        {
+            "setup": {
+                "shelter": {
+                    "pubsub_project_id": "test-project-id",
+                    "sync_repo_topic_id": "test-topic-id",
+                    "enabled": False,
+                }
+            }
+        }
+    )
+
+    publish = mocker.patch("google.cloud.pubsub_v1.PublisherClient.publish")
+
+    # Create new repo with shelter disabled
+    repo = RepositoryFactory(repoid=91728377, name="test-789")
+    dbsession.add(repo)
+    dbsession.commit()
+
+    # Verify no publish was called when shelter is disabled
+    publish.assert_not_called()

database/tests/unit/test_models.py

@@ -191,6 +191,48 @@ def test_upload_token_required_for_public_repos(self, dbsession):
         assert tokens_not_required_owner.onboarding_completed is True
         assert tokens_not_required_owner.upload_token_required_for_public_repos is False
 
+    def test_root_organization(self, dbsession):
+        gitlab_root_group = OwnerFactory.create(
+            username="root_group",
+            service="gitlab",
+            plan="users-pr-inappm",
+        )
+        dbsession.add(gitlab_root_group)
+        gitlab_middle_group = OwnerFactory.create(
+            username="mid_group",
+            service="gitlab",
+            parent_service_id=gitlab_root_group.service_id,
+            root_parent_service_id=None,
+        )
+        dbsession.add(gitlab_middle_group)
+        gitlab_subgroup = OwnerFactory.create(
+            username="subgroup",
+            service="gitlab",
+            parent_service_id=gitlab_middle_group.service_id,
+            root_parent_service_id=None,
+        )
+        dbsession.add(gitlab_subgroup)
+        github_org = OwnerFactory.create(
+            username="gh",
+            service="github",
+        )
+        dbsession.add(github_org)
+        dbsession.flush()
+
+        assert gitlab_root_group.root_organization is None
+        assert gitlab_root_group.root_parent_service_id is None
+
+        assert gitlab_middle_group.root_organization == gitlab_root_group
+        assert (
+            gitlab_middle_group.root_parent_service_id == gitlab_root_group.service_id
+        )
+
+        assert gitlab_subgroup.root_organization == gitlab_root_group
+        assert gitlab_subgroup.root_parent_service_id == gitlab_root_group.service_id
+
+        assert github_org.root_organization is None
+        assert github_org.root_parent_service_id is None
+
 
 class TestAccountModels(object):
     def test_create_account(self, dbsession):

requirements.in

@@ -1,5 +1,5 @@
-https://github.com/codecov/test-results-parser/archive/996ecb2aaf7767bf4c2944c75835c1ee1eb2b566.tar.gz#egg=test-results-parser
-https://github.com/codecov/shared/archive/de4b37bc5a736317c6e7c93f9c58e9ae07f8c96b.tar.gz#egg=shared
+https://github.com/codecov/test-results-parser/archive/190bbc8a911099749928e13d5fe57f6027ca1e74.tar.gz#egg=test-results-parser
+https://github.com/codecov/shared/archive/191837f5e35f5efc410e670aac7e50e0d09e43e1.tar.gz#egg=shared
 https://github.com/codecov/timestring/archive/d37ceacc5954dff3b5bd2f887936a98a668dda42.tar.gz#egg=timestring
 asgiref>=3.7.2
 analytics-python==1.3.0b1
@@ -37,6 +37,7 @@ pytest-celery
 pytest-cov
 pytest-django
 pytest-freezegun
+pytest-insta
 pytest-mock
 pytest-sqlalchemy
 python-dateutil