diff --git a/sdcm/sct_config.py b/sdcm/sct_config.py index e81535c144..fd92cd4c2a 100644 --- a/sdcm/sct_config.py +++ b/sdcm/sct_config.py @@ -2331,10 +2331,11 @@ def _replace_docker_image_latest_tag(self): def _get_target_upgrade_version(self): # 10) update target_upgrade_version automatically - new_scylla_repo = self.get('new_scylla_repo') - if new_scylla_repo and not self.get('target_upgrade_version'): - self['target_upgrade_version'] = get_branch_version(new_scylla_repo) - self.update_argus_with_version(self.get('target_upgrade_version'), "scylla-server-upgrade-target") + if new_scylla_repo := self.get('new_scylla_repo'): + if not self.get('target_upgrade_version'): + self['target_upgrade_version'] = get_branch_version(new_scylla_repo) + scylla_version = get_branch_version(new_scylla_repo, full_version=True) + self.update_argus_with_version(scylla_version, "scylla-server-upgrade-target") def _check_unexpected_sct_variables(self): # check if there are SCT_* environment variable which aren't documented @@ -2591,7 +2592,7 @@ def get_version_based_on_conf(self): # pylint: disable=too-many-locals _is_enterprise = scylla_product == 'scylla-enterprise' elif not self.get('use_preinstalled_scylla'): scylla_repo = self.get('scylla_repo') - scylla_version = get_branch_version(scylla_repo) + scylla_version = get_branch_version(scylla_repo, full_version=True) _is_enterprise = is_enterprise(scylla_version) elif self.get('db_type') == 'cloud_scylla': _is_enterpise = True diff --git a/sdcm/utils/version_utils.py b/sdcm/utils/version_utils.py index 5bcdda3a8a..2a0622b1b9 100644 --- a/sdcm/utils/version_utils.py +++ b/sdcm/utils/version_utils.py @@ -43,7 +43,7 @@ # gemini version 1.0.1, commit ef7c6f422c78ef6b84a6f3bccf52ea9ec846bba0, date 2019-05-16T09:56:16Z GEMINI_VERSION_RE = re.compile(r'\s(?P([\d]+\.[\d]+\.[\d]+)?),') -REPO_VERSIONS_REGEX = re.compile(r'Version: (.*?)\n', re.DOTALL) +REPO_VERSIONS_REGEX = re.compile(r'Filename: .*?server_(.*?)_.*\n', re.DOTALL) # NOTE: following regex is taken from the 'semver' package as is: # https://python-semver.readthedocs.io/en/2.10.0/readme.html @@ -69,7 +69,7 @@ ) SCYLLA_VERSION_RE = re.compile(r"\d+(\.\d+)?\.[\d\w]+([.~][\d\w]+)?") -ARGUS_VERSION_RE = re.compile(r'((?P[\w.~]+)-(0\.)?(?P[0-9]{8,8})\.(?P\w+).*)') +ARGUS_VERSION_RE = re.compile(r'((?P[\w.~]+)(-(0\.)?(?P[0-9]{8,8})?\.(?P\w+).*)?)') SCYLLA_VERSION_GROUPED_RE = re.compile(r'(?P[\w.~]+)-(?P0|rc\d)?\.?(?P[\d]+)\.(?P\w+)') SSTABLE_FORMAT_VERSION_REGEX = re.compile(r'Feature (.*)_SSTABLE_FORMAT is enabled') ENABLED_SSTABLE_FORMAT_VERSION_REGEXP = re.compile(r'(.*)_SSTABLE_FORMAT') @@ -92,7 +92,7 @@ LATEST_SYMLINK_NAME = "latest" NO_TIMESTAMP = dateutil.parser.parse("1961-04-12T06:07:00Z", ignoretz=True) # Poyekhali! -SUPPORTED_PACKAGES = ("scylla", "scylla-enterprise", "scylla-manager") +SUPPORTED_PACKAGES = ("scylla-server", "scylla-enterprise-server", "scylla-manager-server") LOGGER = logging.getLogger(__name__) @@ -172,6 +172,9 @@ def parse(version_string: str): if dotted_build_id_match := re.search(r"(.*\.20[0-9]{6})(\.)([\.\d\w]+)", _scylla_version): _scylla_version = f"{dotted_build_id_match[1]}+{dotted_build_id_match[3]}" + # NOTE: replace '_' with '.' symbol in the build part, example: '3.5.0-dev_0.20250105+ef3b96816_SNAPSHOT + _scylla_version = re.sub(r'_', '.', _scylla_version) + if match := SEMVER_REGEX.match(_scylla_version): return match.groups() raise ValueError( @@ -313,40 +316,51 @@ def get_scylla_urls_from_repository(repo_details): return urls -def get_branch_version_from_debian_repository(urls): +def get_branch_version_from_debian_repository(urls, full_version: bool = False): def get_version(url): data = '\n'.join(get_url_content(url=url)) - # Get only the major version (i.e. "2019.1.1-0.20190709.9f724fedb-1~stretch", get only "2019.1.1") - major_versions = [version.split('-', maxsplit=1)[0] for version in REPO_VERSIONS_REGEX.findall(data)] + if full_version: + major_versions = [version.strip() for version in REPO_VERSIONS_REGEX.findall(data)] + else: + # Get only the major version (i.e. "2019.1.1-0.20190709.9f724fedb-1~stretch", get only "2019.1.1") + major_versions = [version.split('-', maxsplit=1)[0] for version in REPO_VERSIONS_REGEX.findall(data)] if not major_versions: return "" - return max(set(major_versions), key=ComparableScyllaVersion) + return set(major_versions) threads = ParallelObject(objects=urls, timeout=SCYLLA_URL_RESPONSE_TIMEOUT).run(func=get_version) - result = [thread.result for thread in threads] + result = set.union(*[thread.result for thread in threads]) return max(result, key=ComparableScyllaVersion) -def get_branch_version_from_centos_repository(urls): +def get_branch_version_from_centos_repository(urls, full_version: bool = False): def get_version(url): data = '\n'.join(get_url_content(url=url)) primary_path = PRIMARY_XML_GZ_REGEX.search(data).groups()[0] xml_url = url.replace(REPOMD_XML_PATH, primary_path) parser = Parser(url=xml_url) - major_versions = [package['version'][1]['ver'] for package in parser.getList()] - return max(set(major_versions), key=ComparableScyllaVersion) + if full_version: + major_versions = [ + f"{package['version'][1]['ver']}-{package['version'][1]['rel']}" for package in parser.getList() if package['name'][0] in SUPPORTED_PACKAGES] + else: + major_versions = [package['version'][1]['ver'] + for package in parser.getList() if package['name'][0] in SUPPORTED_PACKAGES] + return set(major_versions) threads = ParallelObject(objects=urls, timeout=SCYLLA_URL_RESPONSE_TIMEOUT).run(func=get_version) - result = [thread.result for thread in threads] + result = set.union(*[thread.result for thread in threads]) return max(result, key=ComparableScyllaVersion) -def get_all_versions_from_debian_repository(urls: set[str]) -> set[str]: +def get_all_versions_from_debian_repository(urls: set[str], full_version: bool = False) -> set[str]: def get_version(url: str) -> set[str]: data = '\n'.join(get_url_content(url=url)) - # Get only the major version (i.e. "2019.1.1-0.20190709.9f724fedb-1~stretch", get only "2019.1.1") - major_versions = [version.split('-', maxsplit=1)[0] for version in REPO_VERSIONS_REGEX.findall(data)] + if full_version: + major_versions = [version.strip() for version in REPO_VERSIONS_REGEX.findall(data)] + else: + # Get only the major version (i.e. "2019.1.1-0.20190709.9f724fedb-1~stretch", get only "2019.1.1") + major_versions = [version.split('-', maxsplit=1)[0] for version in REPO_VERSIONS_REGEX.findall(data)] return set(major_versions) threads = ParallelObject(objects=urls, timeout=SCYLLA_URL_RESPONSE_TIMEOUT).run(func=get_version) @@ -354,15 +368,19 @@ def get_version(url: str) -> set[str]: return result -def get_all_versions_from_centos_repository(urls: set[str]) -> set[str]: +def get_all_versions_from_centos_repository(urls: set[str], full_version: bool = False) -> set[str]: def get_version(url: str) -> set[str]: data = '\n'.join(get_url_content(url=url)) primary_path = PRIMARY_XML_GZ_REGEX.search(data).groups()[0] xml_url = url.replace(REPOMD_XML_PATH, primary_path) parser = Parser(url=xml_url) - major_versions = [package['version'][1]['ver'] - for package in parser.getList() if package['name'][0] in SUPPORTED_PACKAGES] + if full_version: + major_versions = [f"{package['version'][1]['ver']}-{package['version'][1]['rel']}" for package in + parser.getList() if package['name'][0] in SUPPORTED_PACKAGES] + else: + major_versions = [package['version'][1]['ver'] + for package in parser.getList() if package['name'][0] in SUPPORTED_PACKAGES] return set(major_versions) threads = ParallelObject(objects=urls, timeout=SCYLLA_URL_RESPONSE_TIMEOUT).run(func=get_version) @@ -382,26 +400,26 @@ def get_repository_details(url): raise ValueError(VERSION_NOT_FOUND_ERROR) -def get_branch_version(url): +def get_branch_version(url, full_version: bool = False): repo_details = get_repository_details(url=url) urls = get_scylla_urls_from_repository(repo_details=repo_details) if repo_details.type == ScyllaFileType.DEBIAN: - return get_branch_version_from_debian_repository(urls=urls) + return get_branch_version_from_debian_repository(urls=urls, full_version=full_version) elif repo_details.type == ScyllaFileType.YUM: - return get_branch_version_from_centos_repository(urls=urls) + return get_branch_version_from_centos_repository(urls=urls, full_version=full_version) # To overcome on Pylint's "inconsistent-return-statements", a value must be returned return [] -def get_all_versions(url: str) -> set[str]: +def get_all_versions(url: str, full_version: bool = False) -> set[str]: repo_details = get_repository_details(url=url) urls = get_scylla_urls_from_repository(repo_details=repo_details) if repo_details.type == ScyllaFileType.DEBIAN: - return get_all_versions_from_debian_repository(urls=urls) + return get_all_versions_from_debian_repository(urls=urls, full_version=full_version) elif repo_details.type == ScyllaFileType.YUM: - return get_all_versions_from_centos_repository(urls=urls) + return get_all_versions_from_centos_repository(urls=urls, full_version=full_version) # To overcome on Pylint's "inconsistent-return-statements", a value must be returned return set() diff --git a/unit_tests/test_version_utils.py b/unit_tests/test_version_utils.py index b834237666..eeb8e5ba37 100644 --- a/unit_tests/test_version_utils.py +++ b/unit_tests/test_version_utils.py @@ -431,6 +431,7 @@ def test_scylla_version_for_argus_regexp(full_version, short, date, commit_id): ("5.2.0-dev-0.20230109.08b3a9c786d9-aarch64", (5, 2, 0, "dev-0.20230109", "08b3a9c786d9")), ("2024.2.0.dev.0.20231219.c7cdb16538f2.1", (2024, 2, 0, "dev-0.20231219", "c7cdb16538f2.1")), ("2024.1.0.rc2.0.20231218.a063c2c16185.1", (2024, 1, 0, "rc2-0.20231218", "a063c2c16185.1")), + ("3.5.0~dev_0.20250105+ef3b96816_SNAPSHOT", (3, 5, 0, "dev.0.20250105", "ef3b96816.SNAPSHOT")), )) def test_comparable_scylla_version_init_positive(version_string, expected): comparable_scylla_version = ComparableScyllaVersion(version_string)