fix(sct_config): scylla-server-upgrade-target wasn't reported to Argus

`ARGUS_VERSION_RE` wasn't covering plain version like `6.3.0~dev`
hence not sending it out to Argus.

the change is fixing the regex

this change also fixes `get_branch_version` to be able to return
a full version string (including git sha and a date), cause
we know Argus would need this data for metrics graphs and perf
summaries.

also not having this information, is casuing confusion on patch
releases, when test are not get to the phase a node is upgrades

Ref: scylladb/argus#533
(cherry picked from commit 7d2d0bb)

sdcm/sct_config.py

@@ -2331,10 +2331,11 @@ def _replace_docker_image_latest_tag(self):
 
     def _get_target_upgrade_version(self):
         # 10) update target_upgrade_version automatically
-        new_scylla_repo = self.get('new_scylla_repo')
-        if new_scylla_repo and not self.get('target_upgrade_version'):
-            self['target_upgrade_version'] = get_branch_version(new_scylla_repo)
-            self.update_argus_with_version(self.get('target_upgrade_version'), "scylla-server-upgrade-target")
+        if new_scylla_repo := self.get('new_scylla_repo'):
+            if not self.get('target_upgrade_version'):
+                self['target_upgrade_version'] = get_branch_version(new_scylla_repo)
+            scylla_version = get_branch_version(new_scylla_repo, full_version=True)
+            self.update_argus_with_version(scylla_version, "scylla-server-upgrade-target")
 
     def _check_unexpected_sct_variables(self):
         # check if there are SCT_* environment variable which aren't documented
@@ -2591,7 +2592,7 @@ def get_version_based_on_conf(self):  # pylint: disable=too-many-locals
                 _is_enterprise = scylla_product == 'scylla-enterprise'
         elif not self.get('use_preinstalled_scylla'):
             scylla_repo = self.get('scylla_repo')
-            scylla_version = get_branch_version(scylla_repo)
+            scylla_version = get_branch_version(scylla_repo, full_version=True)
             _is_enterprise = is_enterprise(scylla_version)
         elif self.get('db_type') == 'cloud_scylla':
             _is_enterpise = True

sdcm/utils/version_utils.py

@@ -43,7 +43,7 @@
 
 # gemini version 1.0.1, commit ef7c6f422c78ef6b84a6f3bccf52ea9ec846bba0, date 2019-05-16T09:56:16Z
 GEMINI_VERSION_RE = re.compile(r'\s(?P<gemini_version>([\d]+\.[\d]+\.[\d]+)?),')
-REPO_VERSIONS_REGEX = re.compile(r'Version: (.*?)\n', re.DOTALL)
+REPO_VERSIONS_REGEX = re.compile(r'Filename: .*?server_(.*?)_.*\n', re.DOTALL)
 
 # NOTE: following regex is taken from the 'semver' package as is:
 #       https://python-semver.readthedocs.io/en/2.10.0/readme.html
@@ -69,7 +69,7 @@
 )
 
 SCYLLA_VERSION_RE = re.compile(r"\d+(\.\d+)?\.[\d\w]+([.~][\d\w]+)?")
-ARGUS_VERSION_RE = re.compile(r'((?P<short>[\w.~]+)-(0\.)?(?P<date>[0-9]{8,8})\.(?P<commit>\w+).*)')
+ARGUS_VERSION_RE = re.compile(r'((?P<short>[\w.~]+)(-(0\.)?(?P<date>[0-9]{8,8})?\.(?P<commit>\w+).*)?)')
 SCYLLA_VERSION_GROUPED_RE = re.compile(r'(?P<version>[\w.~]+)-(?P<build>0|rc\d)?\.?(?P<date>[\d]+)\.(?P<commit_id>\w+)')
 SSTABLE_FORMAT_VERSION_REGEX = re.compile(r'Feature (.*)_SSTABLE_FORMAT is enabled')
 ENABLED_SSTABLE_FORMAT_VERSION_REGEXP = re.compile(r'(.*)_SSTABLE_FORMAT')
@@ -92,7 +92,7 @@
 LATEST_SYMLINK_NAME = "latest"
 NO_TIMESTAMP = dateutil.parser.parse("1961-04-12T06:07:00Z", ignoretz=True)  # Poyekhali!
 
-SUPPORTED_PACKAGES = ("scylla", "scylla-enterprise", "scylla-manager")
+SUPPORTED_PACKAGES = ("scylla-server", "scylla-enterprise-server", "scylla-manager-server")
 
 LOGGER = logging.getLogger(__name__)
 
@@ -172,6 +172,9 @@ def parse(version_string: str):
         if dotted_build_id_match := re.search(r"(.*\.20[0-9]{6})(\.)([\.\d\w]+)", _scylla_version):
             _scylla_version = f"{dotted_build_id_match[1]}+{dotted_build_id_match[3]}"
 
+        # NOTE: replace '_' with '.' symbol in the build part, example: '3.5.0-dev_0.20250105+ef3b96816_SNAPSHOT
+        _scylla_version = re.sub(r'_', '.', _scylla_version)
+
         if match := SEMVER_REGEX.match(_scylla_version):
             return match.groups()
         raise ValueError(
@@ -313,56 +316,71 @@ def get_scylla_urls_from_repository(repo_details):
     return urls
 
 
-def get_branch_version_from_debian_repository(urls):
+def get_branch_version_from_debian_repository(urls, full_version: bool = False):
     def get_version(url):
         data = '\n'.join(get_url_content(url=url))
-        # Get only the major version (i.e. "2019.1.1-0.20190709.9f724fedb-1~stretch", get only "2019.1.1")
-        major_versions = [version.split('-', maxsplit=1)[0] for version in REPO_VERSIONS_REGEX.findall(data)]
+        if full_version:
+            major_versions = [version.strip() for version in REPO_VERSIONS_REGEX.findall(data)]
+        else:
+            # Get only the major version (i.e. "2019.1.1-0.20190709.9f724fedb-1~stretch", get only "2019.1.1")
+            major_versions = [version.split('-', maxsplit=1)[0] for version in REPO_VERSIONS_REGEX.findall(data)]
         if not major_versions:
             return ""
-        return max(set(major_versions), key=ComparableScyllaVersion)
+        return set(major_versions)
 
     threads = ParallelObject(objects=urls, timeout=SCYLLA_URL_RESPONSE_TIMEOUT).run(func=get_version)
-    result = [thread.result for thread in threads]
+    result = set.union(*[thread.result for thread in threads])
     return max(result, key=ComparableScyllaVersion)
 
 
-def get_branch_version_from_centos_repository(urls):
+def get_branch_version_from_centos_repository(urls, full_version: bool = False):
     def get_version(url):
         data = '\n'.join(get_url_content(url=url))
         primary_path = PRIMARY_XML_GZ_REGEX.search(data).groups()[0]
         xml_url = url.replace(REPOMD_XML_PATH, primary_path)
 
         parser = Parser(url=xml_url)
-        major_versions = [package['version'][1]['ver'] for package in parser.getList()]
-        return max(set(major_versions), key=ComparableScyllaVersion)
+        if full_version:
+            major_versions = [
+                f"{package['version'][1]['ver']}-{package['version'][1]['rel']}" for package in parser.getList() if package['name'][0] in SUPPORTED_PACKAGES]
+        else:
+            major_versions = [package['version'][1]['ver']
+                              for package in parser.getList() if package['name'][0] in SUPPORTED_PACKAGES]
+        return set(major_versions)
 
     threads = ParallelObject(objects=urls, timeout=SCYLLA_URL_RESPONSE_TIMEOUT).run(func=get_version)
-    result = [thread.result for thread in threads]
+    result = set.union(*[thread.result for thread in threads])
     return max(result, key=ComparableScyllaVersion)
 
 
-def get_all_versions_from_debian_repository(urls: set[str]) -> set[str]:
+def get_all_versions_from_debian_repository(urls: set[str], full_version: bool = False) -> set[str]:
     def get_version(url: str) -> set[str]:
         data = '\n'.join(get_url_content(url=url))
-        # Get only the major version (i.e. "2019.1.1-0.20190709.9f724fedb-1~stretch", get only "2019.1.1")
-        major_versions = [version.split('-', maxsplit=1)[0] for version in REPO_VERSIONS_REGEX.findall(data)]
+        if full_version:
+            major_versions = [version.strip() for version in REPO_VERSIONS_REGEX.findall(data)]
+        else:
+            # Get only the major version (i.e. "2019.1.1-0.20190709.9f724fedb-1~stretch", get only "2019.1.1")
+            major_versions = [version.split('-', maxsplit=1)[0] for version in REPO_VERSIONS_REGEX.findall(data)]
         return set(major_versions)
 
     threads = ParallelObject(objects=urls, timeout=SCYLLA_URL_RESPONSE_TIMEOUT).run(func=get_version)
     result = set.union(*[thread.result for thread in threads])
     return result
 
 
-def get_all_versions_from_centos_repository(urls: set[str]) -> set[str]:
+def get_all_versions_from_centos_repository(urls: set[str], full_version: bool = False) -> set[str]:
     def get_version(url: str) -> set[str]:
         data = '\n'.join(get_url_content(url=url))
         primary_path = PRIMARY_XML_GZ_REGEX.search(data).groups()[0]
         xml_url = url.replace(REPOMD_XML_PATH, primary_path)
 
         parser = Parser(url=xml_url)
-        major_versions = [package['version'][1]['ver']
-                          for package in parser.getList() if package['name'][0] in SUPPORTED_PACKAGES]
+        if full_version:
+            major_versions = [f"{package['version'][1]['ver']}-{package['version'][1]['rel']}" for package in
+                              parser.getList() if package['name'][0] in SUPPORTED_PACKAGES]
+        else:
+            major_versions = [package['version'][1]['ver']
+                              for package in parser.getList() if package['name'][0] in SUPPORTED_PACKAGES]
         return set(major_versions)
 
     threads = ParallelObject(objects=urls, timeout=SCYLLA_URL_RESPONSE_TIMEOUT).run(func=get_version)
@@ -382,26 +400,26 @@ def get_repository_details(url):
     raise ValueError(VERSION_NOT_FOUND_ERROR)
 
 
-def get_branch_version(url):
+def get_branch_version(url, full_version: bool = False):
     repo_details = get_repository_details(url=url)
     urls = get_scylla_urls_from_repository(repo_details=repo_details)
 
     if repo_details.type == ScyllaFileType.DEBIAN:
-        return get_branch_version_from_debian_repository(urls=urls)
+        return get_branch_version_from_debian_repository(urls=urls, full_version=full_version)
     elif repo_details.type == ScyllaFileType.YUM:
-        return get_branch_version_from_centos_repository(urls=urls)
+        return get_branch_version_from_centos_repository(urls=urls, full_version=full_version)
     # To overcome on Pylint's "inconsistent-return-statements", a value must be returned
     return []
 
 
-def get_all_versions(url: str) -> set[str]:
+def get_all_versions(url: str, full_version: bool = False) -> set[str]:
     repo_details = get_repository_details(url=url)
     urls = get_scylla_urls_from_repository(repo_details=repo_details)
 
     if repo_details.type == ScyllaFileType.DEBIAN:
-        return get_all_versions_from_debian_repository(urls=urls)
+        return get_all_versions_from_debian_repository(urls=urls, full_version=full_version)
     elif repo_details.type == ScyllaFileType.YUM:
-        return get_all_versions_from_centos_repository(urls=urls)
+        return get_all_versions_from_centos_repository(urls=urls, full_version=full_version)
     # To overcome on Pylint's "inconsistent-return-statements", a value must be returned
     return set()
 

unit_tests/test_version_utils.py

@@ -431,6 +431,7 @@ def test_scylla_version_for_argus_regexp(full_version, short, date, commit_id):
     ("5.2.0-dev-0.20230109.08b3a9c786d9-aarch64", (5, 2, 0, "dev-0.20230109", "08b3a9c786d9")),
     ("2024.2.0.dev.0.20231219.c7cdb16538f2.1", (2024, 2, 0, "dev-0.20231219", "c7cdb16538f2.1")),
     ("2024.1.0.rc2.0.20231218.a063c2c16185.1", (2024, 1, 0, "rc2-0.20231218", "a063c2c16185.1")),
+    ("3.5.0~dev_0.20250105+ef3b96816_SNAPSHOT", (3, 5, 0, "dev.0.20250105", "ef3b96816.SNAPSHOT")),
 ))
 def test_comparable_scylla_version_init_positive(version_string, expected):
     comparable_scylla_version = ComparableScyllaVersion(version_string)