From 2aa7f4f0b8797e6e920f387b803efa3dfa530b46 Mon Sep 17 00:00:00 2001 From: Israel Fruchter Date: Thu, 2 Jan 2025 13:47:15 +0200 Subject: [PATCH] fix(sct_config): `scylla-server-upgrade-target` wasn't reported to Argus `ARGUS_VERSION_RE` wasn't covering plain version like `6.3.0~dev` hence not sending it out to Argus. the change is fixing the regex this change also fixes `get_branch_version` to be able to return a full version string (including git sha and a date), cause we know Argus would need this data for metrics graphs and perf summaries. also not having this information, is casuing confusion on patch releases, when test are not get to the phase a node is upgrades Ref: https://github.com/scylladb/argus/issues/533 (cherry picked from commit 7d2d0bba6b6a20bea344cd326b5ed5d37d74fea4) --- sdcm/sct_config.py | 11 +++--- sdcm/utils/version_utils.py | 66 ++++++++++++++++++++------------ unit_tests/test_version_utils.py | 1 + 3 files changed, 49 insertions(+), 29 deletions(-) diff --git a/sdcm/sct_config.py b/sdcm/sct_config.py index e81535c144..fd92cd4c2a 100644 --- a/sdcm/sct_config.py +++ b/sdcm/sct_config.py @@ -2331,10 +2331,11 @@ def _replace_docker_image_latest_tag(self): def _get_target_upgrade_version(self): # 10) update target_upgrade_version automatically - new_scylla_repo = self.get('new_scylla_repo') - if new_scylla_repo and not self.get('target_upgrade_version'): - self['target_upgrade_version'] = get_branch_version(new_scylla_repo) - self.update_argus_with_version(self.get('target_upgrade_version'), "scylla-server-upgrade-target") + if new_scylla_repo := self.get('new_scylla_repo'): + if not self.get('target_upgrade_version'): + self['target_upgrade_version'] = get_branch_version(new_scylla_repo) + scylla_version = get_branch_version(new_scylla_repo, full_version=True) + self.update_argus_with_version(scylla_version, "scylla-server-upgrade-target") def _check_unexpected_sct_variables(self): # check if there are SCT_* environment variable which aren't documented @@ -2591,7 +2592,7 @@ def get_version_based_on_conf(self): # pylint: disable=too-many-locals _is_enterprise = scylla_product == 'scylla-enterprise' elif not self.get('use_preinstalled_scylla'): scylla_repo = self.get('scylla_repo') - scylla_version = get_branch_version(scylla_repo) + scylla_version = get_branch_version(scylla_repo, full_version=True) _is_enterprise = is_enterprise(scylla_version) elif self.get('db_type') == 'cloud_scylla': _is_enterpise = True diff --git a/sdcm/utils/version_utils.py b/sdcm/utils/version_utils.py index 5bcdda3a8a..2a0622b1b9 100644 --- a/sdcm/utils/version_utils.py +++ b/sdcm/utils/version_utils.py @@ -43,7 +43,7 @@ # gemini version 1.0.1, commit ef7c6f422c78ef6b84a6f3bccf52ea9ec846bba0, date 2019-05-16T09:56:16Z GEMINI_VERSION_RE = re.compile(r'\s(?P([\d]+\.[\d]+\.[\d]+)?),') -REPO_VERSIONS_REGEX = re.compile(r'Version: (.*?)\n', re.DOTALL) +REPO_VERSIONS_REGEX = re.compile(r'Filename: .*?server_(.*?)_.*\n', re.DOTALL) # NOTE: following regex is taken from the 'semver' package as is: # https://python-semver.readthedocs.io/en/2.10.0/readme.html @@ -69,7 +69,7 @@ ) SCYLLA_VERSION_RE = re.compile(r"\d+(\.\d+)?\.[\d\w]+([.~][\d\w]+)?") -ARGUS_VERSION_RE = re.compile(r'((?P[\w.~]+)-(0\.)?(?P[0-9]{8,8})\.(?P\w+).*)') +ARGUS_VERSION_RE = re.compile(r'((?P[\w.~]+)(-(0\.)?(?P[0-9]{8,8})?\.(?P\w+).*)?)') SCYLLA_VERSION_GROUPED_RE = re.compile(r'(?P[\w.~]+)-(?P0|rc\d)?\.?(?P[\d]+)\.(?P\w+)') SSTABLE_FORMAT_VERSION_REGEX = re.compile(r'Feature (.*)_SSTABLE_FORMAT is enabled') ENABLED_SSTABLE_FORMAT_VERSION_REGEXP = re.compile(r'(.*)_SSTABLE_FORMAT') @@ -92,7 +92,7 @@ LATEST_SYMLINK_NAME = "latest" NO_TIMESTAMP = dateutil.parser.parse("1961-04-12T06:07:00Z", ignoretz=True) # Poyekhali! -SUPPORTED_PACKAGES = ("scylla", "scylla-enterprise", "scylla-manager") +SUPPORTED_PACKAGES = ("scylla-server", "scylla-enterprise-server", "scylla-manager-server") LOGGER = logging.getLogger(__name__) @@ -172,6 +172,9 @@ def parse(version_string: str): if dotted_build_id_match := re.search(r"(.*\.20[0-9]{6})(\.)([\.\d\w]+)", _scylla_version): _scylla_version = f"{dotted_build_id_match[1]}+{dotted_build_id_match[3]}" + # NOTE: replace '_' with '.' symbol in the build part, example: '3.5.0-dev_0.20250105+ef3b96816_SNAPSHOT + _scylla_version = re.sub(r'_', '.', _scylla_version) + if match := SEMVER_REGEX.match(_scylla_version): return match.groups() raise ValueError( @@ -313,40 +316,51 @@ def get_scylla_urls_from_repository(repo_details): return urls -def get_branch_version_from_debian_repository(urls): +def get_branch_version_from_debian_repository(urls, full_version: bool = False): def get_version(url): data = '\n'.join(get_url_content(url=url)) - # Get only the major version (i.e. "2019.1.1-0.20190709.9f724fedb-1~stretch", get only "2019.1.1") - major_versions = [version.split('-', maxsplit=1)[0] for version in REPO_VERSIONS_REGEX.findall(data)] + if full_version: + major_versions = [version.strip() for version in REPO_VERSIONS_REGEX.findall(data)] + else: + # Get only the major version (i.e. "2019.1.1-0.20190709.9f724fedb-1~stretch", get only "2019.1.1") + major_versions = [version.split('-', maxsplit=1)[0] for version in REPO_VERSIONS_REGEX.findall(data)] if not major_versions: return "" - return max(set(major_versions), key=ComparableScyllaVersion) + return set(major_versions) threads = ParallelObject(objects=urls, timeout=SCYLLA_URL_RESPONSE_TIMEOUT).run(func=get_version) - result = [thread.result for thread in threads] + result = set.union(*[thread.result for thread in threads]) return max(result, key=ComparableScyllaVersion) -def get_branch_version_from_centos_repository(urls): +def get_branch_version_from_centos_repository(urls, full_version: bool = False): def get_version(url): data = '\n'.join(get_url_content(url=url)) primary_path = PRIMARY_XML_GZ_REGEX.search(data).groups()[0] xml_url = url.replace(REPOMD_XML_PATH, primary_path) parser = Parser(url=xml_url) - major_versions = [package['version'][1]['ver'] for package in parser.getList()] - return max(set(major_versions), key=ComparableScyllaVersion) + if full_version: + major_versions = [ + f"{package['version'][1]['ver']}-{package['version'][1]['rel']}" for package in parser.getList() if package['name'][0] in SUPPORTED_PACKAGES] + else: + major_versions = [package['version'][1]['ver'] + for package in parser.getList() if package['name'][0] in SUPPORTED_PACKAGES] + return set(major_versions) threads = ParallelObject(objects=urls, timeout=SCYLLA_URL_RESPONSE_TIMEOUT).run(func=get_version) - result = [thread.result for thread in threads] + result = set.union(*[thread.result for thread in threads]) return max(result, key=ComparableScyllaVersion) -def get_all_versions_from_debian_repository(urls: set[str]) -> set[str]: +def get_all_versions_from_debian_repository(urls: set[str], full_version: bool = False) -> set[str]: def get_version(url: str) -> set[str]: data = '\n'.join(get_url_content(url=url)) - # Get only the major version (i.e. "2019.1.1-0.20190709.9f724fedb-1~stretch", get only "2019.1.1") - major_versions = [version.split('-', maxsplit=1)[0] for version in REPO_VERSIONS_REGEX.findall(data)] + if full_version: + major_versions = [version.strip() for version in REPO_VERSIONS_REGEX.findall(data)] + else: + # Get only the major version (i.e. "2019.1.1-0.20190709.9f724fedb-1~stretch", get only "2019.1.1") + major_versions = [version.split('-', maxsplit=1)[0] for version in REPO_VERSIONS_REGEX.findall(data)] return set(major_versions) threads = ParallelObject(objects=urls, timeout=SCYLLA_URL_RESPONSE_TIMEOUT).run(func=get_version) @@ -354,15 +368,19 @@ def get_version(url: str) -> set[str]: return result -def get_all_versions_from_centos_repository(urls: set[str]) -> set[str]: +def get_all_versions_from_centos_repository(urls: set[str], full_version: bool = False) -> set[str]: def get_version(url: str) -> set[str]: data = '\n'.join(get_url_content(url=url)) primary_path = PRIMARY_XML_GZ_REGEX.search(data).groups()[0] xml_url = url.replace(REPOMD_XML_PATH, primary_path) parser = Parser(url=xml_url) - major_versions = [package['version'][1]['ver'] - for package in parser.getList() if package['name'][0] in SUPPORTED_PACKAGES] + if full_version: + major_versions = [f"{package['version'][1]['ver']}-{package['version'][1]['rel']}" for package in + parser.getList() if package['name'][0] in SUPPORTED_PACKAGES] + else: + major_versions = [package['version'][1]['ver'] + for package in parser.getList() if package['name'][0] in SUPPORTED_PACKAGES] return set(major_versions) threads = ParallelObject(objects=urls, timeout=SCYLLA_URL_RESPONSE_TIMEOUT).run(func=get_version) @@ -382,26 +400,26 @@ def get_repository_details(url): raise ValueError(VERSION_NOT_FOUND_ERROR) -def get_branch_version(url): +def get_branch_version(url, full_version: bool = False): repo_details = get_repository_details(url=url) urls = get_scylla_urls_from_repository(repo_details=repo_details) if repo_details.type == ScyllaFileType.DEBIAN: - return get_branch_version_from_debian_repository(urls=urls) + return get_branch_version_from_debian_repository(urls=urls, full_version=full_version) elif repo_details.type == ScyllaFileType.YUM: - return get_branch_version_from_centos_repository(urls=urls) + return get_branch_version_from_centos_repository(urls=urls, full_version=full_version) # To overcome on Pylint's "inconsistent-return-statements", a value must be returned return [] -def get_all_versions(url: str) -> set[str]: +def get_all_versions(url: str, full_version: bool = False) -> set[str]: repo_details = get_repository_details(url=url) urls = get_scylla_urls_from_repository(repo_details=repo_details) if repo_details.type == ScyllaFileType.DEBIAN: - return get_all_versions_from_debian_repository(urls=urls) + return get_all_versions_from_debian_repository(urls=urls, full_version=full_version) elif repo_details.type == ScyllaFileType.YUM: - return get_all_versions_from_centos_repository(urls=urls) + return get_all_versions_from_centos_repository(urls=urls, full_version=full_version) # To overcome on Pylint's "inconsistent-return-statements", a value must be returned return set() diff --git a/unit_tests/test_version_utils.py b/unit_tests/test_version_utils.py index b834237666..eeb8e5ba37 100644 --- a/unit_tests/test_version_utils.py +++ b/unit_tests/test_version_utils.py @@ -431,6 +431,7 @@ def test_scylla_version_for_argus_regexp(full_version, short, date, commit_id): ("5.2.0-dev-0.20230109.08b3a9c786d9-aarch64", (5, 2, 0, "dev-0.20230109", "08b3a9c786d9")), ("2024.2.0.dev.0.20231219.c7cdb16538f2.1", (2024, 2, 0, "dev-0.20231219", "c7cdb16538f2.1")), ("2024.1.0.rc2.0.20231218.a063c2c16185.1", (2024, 1, 0, "rc2-0.20231218", "a063c2c16185.1")), + ("3.5.0~dev_0.20250105+ef3b96816_SNAPSHOT", (3, 5, 0, "dev.0.20250105", "ef3b96816.SNAPSHOT")), )) def test_comparable_scylla_version_init_positive(version_string, expected): comparable_scylla_version = ComparableScyllaVersion(version_string)