Skip to content

Commit

Permalink
fix(sct_config): scylla-server-upgrade-target wasn't reported to Argus
Browse files Browse the repository at this point in the history
`ARGUS_VERSION_RE` wasn't covering plain version like `6.3.0~dev`
hence not sending it out to Argus.

the change is fixing the regex

this change also fixes `get_branch_version` to be able to return
a full version string (including git sha and a date), cause
we know Argus would need this data for metrics graphs and perf
summaries.

also not having this information, is casuing confusion on patch
releases, when test are not get to the phase a node is upgrades

Ref: scylladb/argus#533
  • Loading branch information
fruch committed Jan 2, 2025
1 parent 4cc0b7f commit a32960c
Show file tree
Hide file tree
Showing 2 changed files with 38 additions and 23 deletions.
11 changes: 6 additions & 5 deletions sdcm/sct_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -2459,10 +2459,11 @@ def _replace_docker_image_latest_tag(self):

def _get_target_upgrade_version(self):
# 10) update target_upgrade_version automatically
new_scylla_repo = self.get('new_scylla_repo')
if new_scylla_repo and not self.get('target_upgrade_version'):
self['target_upgrade_version'] = get_branch_version(new_scylla_repo)
self.update_argus_with_version(self.get('target_upgrade_version'), "scylla-server-upgrade-target")
if new_scylla_repo := self.get('new_scylla_repo'):
if not self.get('target_upgrade_version'):
self['target_upgrade_version'] = get_branch_version(new_scylla_repo)
scylla_version = get_branch_version(new_scylla_repo, full_version=True)
self.update_argus_with_version(scylla_version, "scylla-server-upgrade-target")

def _check_unexpected_sct_variables(self):
# check if there are SCT_* environment variable which aren't documented
Expand Down Expand Up @@ -2723,7 +2724,7 @@ def get_version_based_on_conf(self): # pylint: disable=too-many-locals
_is_enterprise = scylla_product == 'scylla-enterprise'
elif not self.get('use_preinstalled_scylla'):
scylla_repo = self.get('scylla_repo')
scylla_version = get_branch_version(scylla_repo)
scylla_version = get_branch_version(scylla_repo, full_version=True)
_is_enterprise = is_enterprise(scylla_version)
elif self.get('db_type') == 'cloud_scylla':
_is_enterpise = True
Expand Down
50 changes: 32 additions & 18 deletions sdcm/utils/version_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@
)

SCYLLA_VERSION_RE = re.compile(r"\d+(\.\d+)?\.[\d\w]+([.~][\d\w]+)?")
ARGUS_VERSION_RE = re.compile(r'((?P<short>[\w.~]+)-(0\.)?(?P<date>[0-9]{8,8})\.(?P<commit>\w+).*)')
ARGUS_VERSION_RE = re.compile(r'((?P<short>[\w.~]+)(-(0\.)?(?P<date>[0-9]{8,8})?\.(?P<commit>\w+).*)?)')
SCYLLA_VERSION_GROUPED_RE = re.compile(r'(?P<version>[\w.~]+)-(?P<build>0|rc\d)?\.?(?P<date>[\d]+)\.(?P<commit_id>\w+)')
SSTABLE_FORMAT_VERSION_REGEX = re.compile(r'Feature (.*)_SSTABLE_FORMAT is enabled')
ENABLED_SSTABLE_FORMAT_VERSION_REGEXP = re.compile(r'(.*)_SSTABLE_FORMAT')
Expand Down Expand Up @@ -313,11 +313,14 @@ def get_scylla_urls_from_repository(repo_details):
return urls


def get_branch_version_from_debian_repository(urls):
def get_branch_version_from_debian_repository(urls, full_version: bool = False):
def get_version(url):
data = '\n'.join(get_url_content(url=url))
# Get only the major version (i.e. "2019.1.1-0.20190709.9f724fedb-1~stretch", get only "2019.1.1")
major_versions = [version.split('-', maxsplit=1)[0] for version in REPO_VERSIONS_REGEX.findall(data)]
if full_version:
major_versions = [version.strip() for version in REPO_VERSIONS_REGEX.findall(data)]
else:
# Get only the major version (i.e. "2019.1.1-0.20190709.9f724fedb-1~stretch", get only "2019.1.1")
major_versions = [version.split('-', maxsplit=1)[0] for version in REPO_VERSIONS_REGEX.findall(data)]
if not major_versions:
return ""
return max(set(major_versions), key=ComparableScyllaVersion)
Expand All @@ -327,42 +330,53 @@ def get_version(url):
return max(result, key=ComparableScyllaVersion)


def get_branch_version_from_centos_repository(urls):
def get_branch_version_from_centos_repository(urls, full_version: bool = False):
def get_version(url):
data = '\n'.join(get_url_content(url=url))
primary_path = PRIMARY_XML_GZ_REGEX.search(data).groups()[0]
xml_url = url.replace(REPOMD_XML_PATH, primary_path)

parser = Parser(url=xml_url)
major_versions = [package['version'][1]['ver'] for package in parser.getList()]
if full_version:
major_versions = [
f"{package['version'][1]['ver']}-{package['version'][1]['rel']}" for package in parser.getList()]
else:
major_versions = [package['version'][1]['ver'] for package in parser.getList()]
return max(set(major_versions), key=ComparableScyllaVersion)

threads = ParallelObject(objects=urls, timeout=SCYLLA_URL_RESPONSE_TIMEOUT).run(func=get_version)
result = [thread.result for thread in threads]
return max(result, key=ComparableScyllaVersion)


def get_all_versions_from_debian_repository(urls: set[str]) -> set[str]:
def get_all_versions_from_debian_repository(urls: set[str], full_version: bool = False) -> set[str]:
def get_version(url: str) -> set[str]:
data = '\n'.join(get_url_content(url=url))
# Get only the major version (i.e. "2019.1.1-0.20190709.9f724fedb-1~stretch", get only "2019.1.1")
major_versions = [version.split('-', maxsplit=1)[0] for version in REPO_VERSIONS_REGEX.findall(data)]
if full_version:
major_versions = [version.strip() for version in REPO_VERSIONS_REGEX.findall(data)]
else:
# Get only the major version (i.e. "2019.1.1-0.20190709.9f724fedb-1~stretch", get only "2019.1.1")
major_versions = [version.split('-', maxsplit=1)[0] for version in REPO_VERSIONS_REGEX.findall(data)]
return set(major_versions)

threads = ParallelObject(objects=urls, timeout=SCYLLA_URL_RESPONSE_TIMEOUT).run(func=get_version)
result = set.union(*[thread.result for thread in threads])
return result


def get_all_versions_from_centos_repository(urls: set[str]) -> set[str]:
def get_all_versions_from_centos_repository(urls: set[str], full_version: bool = False) -> set[str]:
def get_version(url: str) -> set[str]:
data = '\n'.join(get_url_content(url=url))
primary_path = PRIMARY_XML_GZ_REGEX.search(data).groups()[0]
xml_url = url.replace(REPOMD_XML_PATH, primary_path)

parser = Parser(url=xml_url)
major_versions = [package['version'][1]['ver']
for package in parser.getList() if package['name'][0] in SUPPORTED_PACKAGES]
if full_version:
major_versions = [f"{package['version'][1]['ver']}-{package['version'][1]['rel']}" for package in
parser.getList()]
else:
major_versions = [package['version'][1]['ver']
for package in parser.getList() if package['name'][0] in SUPPORTED_PACKAGES]
return set(major_versions)

threads = ParallelObject(objects=urls, timeout=SCYLLA_URL_RESPONSE_TIMEOUT).run(func=get_version)
Expand All @@ -382,26 +396,26 @@ def get_repository_details(url):
raise ValueError(VERSION_NOT_FOUND_ERROR)


def get_branch_version(url):
def get_branch_version(url, full_version: bool = False):
repo_details = get_repository_details(url=url)
urls = get_scylla_urls_from_repository(repo_details=repo_details)

if repo_details.type == ScyllaFileType.DEBIAN:
return get_branch_version_from_debian_repository(urls=urls)
return get_branch_version_from_debian_repository(urls=urls, full_version=full_version)
elif repo_details.type == ScyllaFileType.YUM:
return get_branch_version_from_centos_repository(urls=urls)
return get_branch_version_from_centos_repository(urls=urls, full_version=full_version)
# To overcome on Pylint's "inconsistent-return-statements", a value must be returned
return []


def get_all_versions(url: str) -> set[str]:
def get_all_versions(url: str, full_version: bool = False) -> set[str]:
repo_details = get_repository_details(url=url)
urls = get_scylla_urls_from_repository(repo_details=repo_details)

if repo_details.type == ScyllaFileType.DEBIAN:
return get_all_versions_from_debian_repository(urls=urls)
return get_all_versions_from_debian_repository(urls=urls, full_version=full_version)
elif repo_details.type == ScyllaFileType.YUM:
return get_all_versions_from_centos_repository(urls=urls)
return get_all_versions_from_centos_repository(urls=urls, full_version=full_version)
# To overcome on Pylint's "inconsistent-return-statements", a value must be returned
return set()

Expand Down

0 comments on commit a32960c

Please sign in to comment.