[Backport perf-v16] fix(sct_config): scylla-server-upgrade-target wasn't reported to Argus

sdcm/sct_config.py

@@ -2331,10 +2331,11 @@ def _replace_docker_image_latest_tag(self):
 
     def _get_target_upgrade_version(self):
         # 10) update target_upgrade_version automatically
-        new_scylla_repo = self.get('new_scylla_repo')
-        if new_scylla_repo and not self.get('target_upgrade_version'):
-            self['target_upgrade_version'] = get_branch_version(new_scylla_repo)
-            self.update_argus_with_version(self.get('target_upgrade_version'), "scylla-server-upgrade-target")
+        if new_scylla_repo := self.get('new_scylla_repo'):
+            if not self.get('target_upgrade_version'):
+                self['target_upgrade_version'] = get_branch_version(new_scylla_repo)
+            scylla_version = get_branch_version(new_scylla_repo, full_version=True)
+            self.update_argus_with_version(scylla_version, "scylla-server-upgrade-target")
 
     def _check_unexpected_sct_variables(self):
         # check if there are SCT_* environment variable which aren't documented
@@ -2591,7 +2592,7 @@ def get_version_based_on_conf(self):  # pylint: disable=too-many-locals
                 _is_enterprise = scylla_product == 'scylla-enterprise'
         elif not self.get('use_preinstalled_scylla'):
             scylla_repo = self.get('scylla_repo')
-            scylla_version = get_branch_version(scylla_repo)
+            scylla_version = get_branch_version(scylla_repo, full_version=True)
             _is_enterprise = is_enterprise(scylla_version)
         elif self.get('db_type') == 'cloud_scylla':
             _is_enterpise = True

sdcm/utils/version_utils.py

@@ -43,7 +43,7 @@
 
 # gemini version 1.0.1, commit ef7c6f422c78ef6b84a6f3bccf52ea9ec846bba0, date 2019-05-16T09:56:16Z
 GEMINI_VERSION_RE = re.compile(r'\s(?P<gemini_version>([\d]+\.[\d]+\.[\d]+)?),')
-REPO_VERSIONS_REGEX = re.compile(r'Version: (.*?)\n', re.DOTALL)
+REPO_VERSIONS_REGEX = re.compile(r'Filename: .*?server_(.*?)_.*\n', re.DOTALL)
 
 # NOTE: following regex is taken from the 'semver' package as is:
 #       https://python-semver.readthedocs.io/en/2.10.0/readme.html
@@ -69,7 +69,7 @@
 )
 
 SCYLLA_VERSION_RE = re.compile(r"\d+(\.\d+)?\.[\d\w]+([.~][\d\w]+)?")
-ARGUS_VERSION_RE = re.compile(r'((?P<short>[\w.~]+)-(0\.)?(?P<date>[0-9]{8,8})\.(?P<commit>\w+).*)')
+ARGUS_VERSION_RE = re.compile(r'((?P<short>[\w.~]+)(-(0\.)?(?P<date>[0-9]{8,8})?\.(?P<commit>\w+).*)?)')
 SCYLLA_VERSION_GROUPED_RE = re.compile(r'(?P<version>[\w.~]+)-(?P<build>0|rc\d)?\.?(?P<date>[\d]+)\.(?P<commit_id>\w+)')
 SSTABLE_FORMAT_VERSION_REGEX = re.compile(r'Feature (.*)_SSTABLE_FORMAT is enabled')
 ENABLED_SSTABLE_FORMAT_VERSION_REGEXP = re.compile(r'(.*)_SSTABLE_FORMAT')
@@ -92,7 +92,7 @@
 LATEST_SYMLINK_NAME = "latest"
 NO_TIMESTAMP = dateutil.parser.parse("1961-04-12T06:07:00Z", ignoretz=True)  # Poyekhali!
 
-SUPPORTED_PACKAGES = ("scylla", "scylla-enterprise", "scylla-manager")
+SUPPORTED_PACKAGES = ("scylla-server", "scylla-enterprise-server", "scylla-manager-server")
 
 LOGGER = logging.getLogger(__name__)
 
@@ -172,6 +172,9 @@ def parse(version_string: str):
         if dotted_build_id_match := re.search(r"(.*\.20[0-9]{6})(\.)([\.\d\w]+)", _scylla_version):
             _scylla_version = f"{dotted_build_id_match[1]}+{dotted_build_id_match[3]}"
 
+        # NOTE: replace '_' with '.' symbol in the build part, example: '3.5.0-dev_0.20250105+ef3b96816_SNAPSHOT
+        _scylla_version = re.sub(r'_', '.', _scylla_version)
+
         if match := SEMVER_REGEX.match(_scylla_version):
             return match.groups()
         raise ValueError(
@@ -313,56 +316,71 @@ def get_scylla_urls_from_repository(repo_details):
     return urls
 
 
-def get_branch_version_from_debian_repository(urls):
+def get_branch_version_from_debian_repository(urls, full_version: bool = False):
     def get_version(url):
         data = '\n'.join(get_url_content(url=url))
-        # Get only the major version (i.e. "2019.1.1-0.20190709.9f724fedb-1~stretch", get only "2019.1.1")
-        major_versions = [version.split('-', maxsplit=1)[0] for version in REPO_VERSIONS_REGEX.findall(data)]
+        if full_version:
+            major_versions = [version.strip() for version in REPO_VERSIONS_REGEX.findall(data)]
+        else:
+            # Get only the major version (i.e. "2019.1.1-0.20190709.9f724fedb-1~stretch", get only "2019.1.1")
+            major_versions = [version.split('-', maxsplit=1)[0] for version in REPO_VERSIONS_REGEX.findall(data)]
         if not major_versions:
             return ""
-        return max(set(major_versions), key=ComparableScyllaVersion)
+        return set(major_versions)
 
     threads = ParallelObject(objects=urls, timeout=SCYLLA_URL_RESPONSE_TIMEOUT).run(func=get_version)
-    result = [thread.result for thread in threads]
+    result = set.union(*[thread.result for thread in threads])
     return max(result, key=ComparableScyllaVersion)
 
 
-def get_branch_version_from_centos_repository(urls):
+def get_branch_version_from_centos_repository(urls, full_version: bool = False):
     def get_version(url):
         data = '\n'.join(get_url_content(url=url))
         primary_path = PRIMARY_XML_GZ_REGEX.search(data).groups()[0]
         xml_url = url.replace(REPOMD_XML_PATH, primary_path)
 
         parser = Parser(url=xml_url)
-        major_versions = [package['version'][1]['ver'] for package in parser.getList()]
-        return max(set(major_versions), key=ComparableScyllaVersion)
+        if full_version:
+            major_versions = [
+                f"{package['version'][1]['ver']}-{package['version'][1]['rel']}" for package in parser.getList() if package['name'][0] in SUPPORTED_PACKAGES]
+        else:
+            major_versions = [package['version'][1]['ver']
+                              for package in parser.getList() if package['name'][0] in SUPPORTED_PACKAGES]
+        return set(major_versions)
 
     threads = ParallelObject(objects=urls, timeout=SCYLLA_URL_RESPONSE_TIMEOUT).run(func=get_version)
-    result = [thread.result for thread in threads]
+    result = set.union(*[thread.result for thread in threads])
     return max(result, key=ComparableScyllaVersion)
 
 
-def get_all_versions_from_debian_repository(urls: set[str]) -> set[str]:
+def get_all_versions_from_debian_repository(urls: set[str], full_version: bool = False) -> set[str]:
     def get_version(url: str) -> set[str]:
         data = '\n'.join(get_url_content(url=url))
-        # Get only the major version (i.e. "2019.1.1-0.20190709.9f724fedb-1~stretch", get only "2019.1.1")
-        major_versions = [version.split('-', maxsplit=1)[0] for version in REPO_VERSIONS_REGEX.findall(data)]
+        if full_version:
+            major_versions = [version.strip() for version in REPO_VERSIONS_REGEX.findall(data)]
+        else:
+            # Get only the major version (i.e. "2019.1.1-0.20190709.9f724fedb-1~stretch", get only "2019.1.1")
+            major_versions = [version.split('-', maxsplit=1)[0] for version in REPO_VERSIONS_REGEX.findall(data)]
         return set(major_versions)
 
     threads = ParallelObject(objects=urls, timeout=SCYLLA_URL_RESPONSE_TIMEOUT).run(func=get_version)
     result = set.union(*[thread.result for thread in threads])
     return result
 
 
-def get_all_versions_from_centos_repository(urls: set[str]) -> set[str]:
+def get_all_versions_from_centos_repository(urls: set[str], full_version: bool = False) -> set[str]:
     def get_version(url: str) -> set[str]:
         data = '\n'.join(get_url_content(url=url))
         primary_path = PRIMARY_XML_GZ_REGEX.search(data).groups()[0]
         xml_url = url.replace(REPOMD_XML_PATH, primary_path)
 
         parser = Parser(url=xml_url)
-        major_versions = [package['version'][1]['ver']
-                          for package in parser.getList() if package['name'][0] in SUPPORTED_PACKAGES]
+        if full_version:
+            major_versions = [f"{package['version'][1]['ver']}-{package['version'][1]['rel']}" for package in
+                              parser.getList() if package['name'][0] in SUPPORTED_PACKAGES]
+        else:
+            major_versions = [package['version'][1]['ver']
+                              for package in parser.getList() if package['name'][0] in SUPPORTED_PACKAGES]
         return set(major_versions)
 
     threads = ParallelObject(objects=urls, timeout=SCYLLA_URL_RESPONSE_TIMEOUT).run(func=get_version)
@@ -382,26 +400,26 @@ def get_repository_details(url):
     raise ValueError(VERSION_NOT_FOUND_ERROR)
 
 
-def get_branch_version(url):
+def get_branch_version(url, full_version: bool = False):
     repo_details = get_repository_details(url=url)
     urls = get_scylla_urls_from_repository(repo_details=repo_details)
 
     if repo_details.type == ScyllaFileType.DEBIAN:
-        return get_branch_version_from_debian_repository(urls=urls)
+        return get_branch_version_from_debian_repository(urls=urls, full_version=full_version)
     elif repo_details.type == ScyllaFileType.YUM:
-        return get_branch_version_from_centos_repository(urls=urls)
+        return get_branch_version_from_centos_repository(urls=urls, full_version=full_version)
     # To overcome on Pylint's "inconsistent-return-statements", a value must be returned
     return []
 
 
-def get_all_versions(url: str) -> set[str]:
+def get_all_versions(url: str, full_version: bool = False) -> set[str]:
     repo_details = get_repository_details(url=url)
     urls = get_scylla_urls_from_repository(repo_details=repo_details)
 
     if repo_details.type == ScyllaFileType.DEBIAN:
-        return get_all_versions_from_debian_repository(urls=urls)
+        return get_all_versions_from_debian_repository(urls=urls, full_version=full_version)
     elif repo_details.type == ScyllaFileType.YUM:
-        return get_all_versions_from_centos_repository(urls=urls)
+        return get_all_versions_from_centos_repository(urls=urls, full_version=full_version)
     # To overcome on Pylint's "inconsistent-return-statements", a value must be returned
     return set()
 

unit_tests/test_version_utils.py

@@ -431,6 +431,7 @@ def test_scylla_version_for_argus_regexp(full_version, short, date, commit_id):
     ("5.2.0-dev-0.20230109.08b3a9c786d9-aarch64", (5, 2, 0, "dev-0.20230109", "08b3a9c786d9")),
     ("2024.2.0.dev.0.20231219.c7cdb16538f2.1", (2024, 2, 0, "dev-0.20231219", "c7cdb16538f2.1")),
     ("2024.1.0.rc2.0.20231218.a063c2c16185.1", (2024, 1, 0, "rc2-0.20231218", "a063c2c16185.1")),
+    ("3.5.0~dev_0.20250105+ef3b96816_SNAPSHOT", (3, 5, 0, "dev.0.20250105", "ef3b96816.SNAPSHOT")),
 ))
 def test_comparable_scylla_version_init_positive(version_string, expected):
     comparable_scylla_version = ComparableScyllaVersion(version_string)