diff --git a/.flake8 b/.flake8 index 58c10800..8afc16eb 100644 --- a/.flake8 +++ b/.flake8 @@ -1,4 +1,4 @@ [flake8] max-line-length = 160 -ignore = W504, F401, F811 +ignore = W504, F401, F811, E501 exclude = powerapi/test_utils \ No newline at end of file diff --git a/src/powerapi/report/power_report.py b/src/powerapi/report/power_report.py index ed96fa12..4db14f0d 100644 --- a/src/powerapi/report/power_report.py +++ b/src/powerapi/report/power_report.py @@ -151,32 +151,31 @@ def to_virtiofs_db(report: PowerReport) -> Tuple[str, str]: power = report.power return filename, power - def gen_tag(self, metadata_kept): + def generate_tags(self, selected_tags: None | list[str] = None) -> dict[str, Any]: """ - Generate the tags list of the report. - :param metadata_kept: The metadata to keep + Generate the report tags from its metadata. + :param selected_tags: List of tags to be included (in flattened/sanitized form), None to include everything + :return: a single level dictionary containing the tags of the report """ - # Always sensor and target are kept - tags = {'sensor': self.sensor, 'target': self.target} + flattened_tags = self.flatten_tags(self.metadata) + sanitized_tags_name = self.sanitize_tags_name(flattened_tags) + sanitized_tags = {sanitized_tags_name[k]: v for k, v in flattened_tags.items()} - if metadata_kept: - for metadata_name in metadata_kept: - if metadata_name not in self.metadata: - raise BadInputData(f'No tag "{metadata_name}" found in power report', self) - tags[metadata_name] = self.metadata[metadata_name] + if selected_tags: + tags = {k: v for k, v in sanitized_tags.items() if k in selected_tags} else: - tags.update(self.metadata) + tags = sanitized_tags - return tags + return {'sensor': self.sensor, 'target': self.target} | tags @staticmethod - def to_influxdb(report: PowerReport, tags: List[str]) -> Dict: + def to_influxdb(report: PowerReport, tags: None | list[str]) -> dict[str, Any]: """ :return: a dictionary, that can be stored into an influxdb, from a given PowerReport """ return { 'measurement': 'power_consumption', - 'tags': report.gen_tag(tags), + 'tags': report.generate_tags(tags), 'time': str(report.timestamp), 'fields': { 'power': report.power @@ -184,12 +183,12 @@ def to_influxdb(report: PowerReport, tags: List[str]) -> Dict: } @staticmethod - def to_prometheus(report: PowerReport, tags: List[str]) -> Dict: + def to_prometheus(report: PowerReport, tags: None | list[str]) -> dict[str, Any]: """ :return: a dictionary, that can be stored into a prometheus instance, from a given PowerReport """ return { - 'tags': report.gen_tag(tags), + 'tags': report.generate_tags(tags), 'time': int(report.timestamp.timestamp()), 'value': report.power } diff --git a/src/powerapi/report/report.py b/src/powerapi/report/report.py index b5e79b28..4b2d43d0 100644 --- a/src/powerapi/report/report.py +++ b/src/powerapi/report/report.py @@ -29,8 +29,11 @@ from __future__ import annotations +from collections import Counter from datetime import datetime -from typing import Dict, NewType, Tuple, List, Any +from typing import Dict, NewType, Tuple, List, Any, Iterable +from zlib import crc32 + from powerapi.exception import PowerAPIExceptionWithMessage, PowerAPIException from powerapi.message import Message @@ -43,6 +46,8 @@ CSV_HEADER_COMMON = [TIMESTAMP_KEY, SENSOR_KEY, TARGET_KEY] CsvLines = NewType('CsvLines', Tuple[List[str], Dict[str, str]]) +TAGS_NAME_TRANSLATION_TABLE = str.maketrans('.-/', '___') + class BadInputData(PowerAPIExceptionWithMessage): """ @@ -133,3 +138,40 @@ def create_empty_report(): Creates an empty report """ return Report(None, None, None) + + @staticmethod + def sanitize_tags_name(tags: Iterable[str]) -> dict[str, str]: + """ + Generate a dict containing the tags name and theirs corresponding sanitized version. + + The tags name are sanitized according to InfluxDB and Prometheus restrictions. + If a sanitized tag have conflicts (`tag-name` and `tag.name` -> `tag_name`) a hash of the input tag will be + appended at the end of the sanitized tag name. This allows to have stable tags name in the destination database. + :param tags: Iterable object containing the tags name + :return: Dictionary containing the input tag name as key and its sanitized version as value + """ + sanitized_tags = {tag: tag.translate(TAGS_NAME_TRANSLATION_TABLE) for tag in tags} + conflict_count = Counter(sanitized_tags.values()) + return { + tag_orig: (tag_new if conflict_count[tag_new] == 1 else f'{tag_new}_{crc32(tag_orig.encode()):x}') + for tag_orig, tag_new in sanitized_tags.items() + } + + @staticmethod + def flatten_tags(tags: dict[str, Any], separator: str = '_') -> dict[str, Any]: + """ + Flatten nested dictionaries within a tags dictionary. + + This method takes a dictionary of tags, which may contain nested dictionaries as values, and flattens them into + a single-level dictionary. Each key in the flattened dictionary is constructed by concatenating the keys from + the nested dictionaries with their parent keys, separated by the specified separator. + + This is particularly useful for databases that only support canonical (non-nested) types as values. + :param tags: Input tags dict + :param separator: Separator to use for the flattened tags name + :return: Flattened tags dict + """ + return { + f"{pkey}{separator}{ckey}" if isinstance(pvalue, dict) else pkey: cvalue for pkey, pvalue in tags.items() + for ckey, cvalue in (pvalue.items() if isinstance(pvalue, dict) else {pkey: pvalue}.items()) + } diff --git a/tests/unit/report/conftest.py b/tests/unit/report/conftest.py index 2fb92595..39c45505 100644 --- a/tests/unit/report/conftest.py +++ b/tests/unit/report/conftest.py @@ -26,10 +26,11 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from datetime import datetime + import pytest from powerapi.report import PowerReport -from tests.utils.report.power import gen_json_power_report @pytest.fixture @@ -37,36 +38,56 @@ def power_report_without_metadata() -> PowerReport: """ Generates a power_power """ - json_input = gen_json_power_report(1)[0] - report = PowerReport.from_json(json_input) - - return report + ts = datetime(2020, 1, 1, 0, 0, 0) + sensor = 'pytest' + target = 'test' + power = 42 + metadata = {} + return PowerReport(ts, sensor, target, power, metadata) @pytest.fixture def power_report_with_metadata(power_report_without_metadata) -> PowerReport: """ - Generates a power_power + Generates a power report with single-level metadata. """ - power_report_without_metadata.metadata = {'k1': 'v1', - 'k2': 'v2', - 'k3': 333, - 'k4': 'vv4'} - + power_report_without_metadata.metadata = { + 'scope': 'cpu', + 'socket': 0, + 'formula': '0000000000000000000000000000000000000000' + } return power_report_without_metadata @pytest.fixture -def power_report_with_nested_metadata(power_report_without_metadata) -> PowerReport: +def power_report_with_metadata_expected_tags(power_report_with_metadata) -> set[str]: """ - Generates a power_power + Returns the expected tags for the power report with single-level metadata. """ - power_report_without_metadata.metadata = {'k1': {'k1_k1': 1}, - 'k2': 'v2', - 'k3': 333, - 'k4': {'k4_k1': 'v1', - 'k4_k2': {'k4_k2_k1': 'v2'} - } - } + return {'sensor', 'target', 'scope', 'socket', 'formula'} + +@pytest.fixture +def power_report_with_nested_metadata(power_report_without_metadata) -> PowerReport: + """ + Generates a power report with nested metadata. + """ + power_report_without_metadata.metadata = { + 'scope': 'cpu', + 'socket': 0, + 'formula': '0000000000000000000000000000000000000000', + 'k8s': { + 'app.kubernetes.io/name': 'test', + 'app.kubernetes.io/managed-by': 'pytest', + 'helm.sh/chart': 'powerapi-pytest-1.0.0' + } + } return power_report_without_metadata + + +@pytest.fixture +def power_report_with_nested_metadata_expected_tags(power_report_with_nested_metadata) -> set[str]: + """ + Returns the expected tags for the power report with nested metadata. + """ + return {'sensor', 'target', 'scope', 'socket', 'formula', 'k8s_app_kubernetes_io_name', 'k8s_app_kubernetes_io_managed_by', 'k8s_helm_sh_chart'} diff --git a/tests/unit/report/test_power_report.py b/tests/unit/report/test_power_report.py index 6e8d63e8..dbb9e65d 100644 --- a/tests/unit/report/test_power_report.py +++ b/tests/unit/report/test_power_report.py @@ -1,21 +1,21 @@ -# Copyright (c) 2021, INRIA +# Copyright (c) 2021, Inria # Copyright (c) 2021, University of Lille # All rights reserved. - +# # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: - +# # * Redistributions of source code must retain the above copyright notice, this # list of conditions and the following disclaimer. - +# # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. - +# # * Neither the name of the copyright holder nor the names of its # contributors may be used to endorse or promote products derived from # this software without specific prior written permission. - +# # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE @@ -61,49 +61,6 @@ def get_expected_metadata_power_report_with_tags(report: PowerReport, tags: list return metadata -def get_expected_influxdb_document(report: PowerReport, tags: list) -> dict: - """ - Generates a dictionary that represents the expected influxdb document for a given dictionary - :param report: The report for generating the document - :param tags: The tags to be kept - """ - return { - 'measurement': 'power_consumption', - 'tags': get_expected_metadata_power_report_with_tags(report, tags) if tags else - get_expected_metadata_power_report_without_tag_list(report), - 'time': str(report.timestamp), - 'fields': { - 'power': report.power - } - } - - -def get_expected_prometheus_document(report: PowerReport, tags: list) -> dict: - """ - Generates a dictionary that represents the expected prometheus document for a given dictionary - :param report: The report for generating the document - :param tags: The tags to be kept - """ - return { - 'tags': get_expected_metadata_power_report_with_tags(report, tags) if tags else - get_expected_metadata_power_report_without_tag_list(report), - 'time': int(report.timestamp.timestamp()), - 'value': report.power - } - - -def check_report_metadata(original_metadata: dict, report: PowerReport): - """ - Check that the metadata of a report didn't change - :param original_metadata: Orignal's report metadata - :param report: Report for checking metadata - """ - assert report.metadata == original_metadata - - -######## -# JSON # -######## def test_create_power_report_from_json_wit_str_timestamp_create_a_PowerReport(): json_input = gen_json_power_report(1)[0] report = PowerReport.from_json(json_input) @@ -138,9 +95,6 @@ def test_create_power_report_from_json_without_sensor_field_raise_BadInputData() _ = PowerReport.from_json(json_input) -####### -# CSV # -####### def test_create_power_report_from_csv_with_one_lines_create_an_power_report(): csv_lines = [("power", { @@ -191,10 +145,6 @@ def test_create_power_report_from_csv_with_two_lines_raise_BadInputData(): _ = PowerReport.from_csv_lines(csv_lines) -############ -# METADATA # -############ - def test_creating_report_with_metadata(): report = PowerReport(('1970-09-01T09:09:10.543'), 'toto', 'all', 42, {"tag": 1}) assert report.metadata["tag"] == 1 @@ -223,323 +173,141 @@ def test_create_report_from_csv_with_metadata(): assert report.metadata["tag"] == 1 -def test_gen_tag_keep_all_the_report_metadata_without_tags_list_and_empty_metadata(power_report_without_metadata): - tags = [] - original_metadata = power_report_without_metadata.metadata - expected_metadata = get_expected_metadata_power_report_without_tag_list(power_report_without_metadata) - - metadata = power_report_without_metadata.gen_tag(tags) - - assert metadata == expected_metadata - check_report_metadata(original_metadata, power_report_without_metadata) - - -def test_gen_tag_keep_all_the_report_metadata_without_tag_list_and_empty_metadata(power_report_without_metadata): - tags = None - original_metadata = power_report_without_metadata.metadata - expected_metadata = get_expected_metadata_power_report_without_tag_list(power_report_without_metadata) - - metadata = power_report_without_metadata.gen_tag(tags) - - assert metadata == expected_metadata - check_report_metadata(original_metadata, power_report_without_metadata) - - -def test_gen_tag_keep_all_the_report_metadata_with_empty_tag_list(power_report_with_metadata): - tags = [] - original_metadata = power_report_with_metadata.metadata - expected_metadata = get_expected_metadata_power_report_without_tag_list(power_report_with_metadata) - - metadata = power_report_with_metadata.gen_tag(tags) - - assert metadata == expected_metadata - check_report_metadata(original_metadata, power_report_with_metadata) - - -def test_gen_tag_keep_all_the_report_metadata_without_tags(power_report_with_metadata): - tags = None - original_metadata = power_report_with_metadata.metadata - expected_metadata = get_expected_metadata_power_report_without_tag_list(power_report_with_metadata) - - metadata = power_report_with_metadata.gen_tag(tags) - - assert metadata == expected_metadata - check_report_metadata(original_metadata, power_report_with_metadata) - - -def test_gen_tag_keep_all_the_report_nested_metadata_with_empty_tag_list(power_report_with_nested_metadata): - tags = [] - original_metadata = power_report_with_nested_metadata.metadata - expected_metadata = get_expected_metadata_power_report_without_tag_list(power_report_with_nested_metadata) - - metadata = power_report_with_nested_metadata.gen_tag(tags) - - assert metadata == expected_metadata - check_report_metadata(original_metadata, power_report_with_nested_metadata) - - -def test_gen_tag_keep_all_the_report_nested_metadata_without_tags(power_report_with_nested_metadata): - tags = None - original_metadata = power_report_with_nested_metadata.metadata - expected_metadata = get_expected_metadata_power_report_without_tag_list(power_report_with_nested_metadata) - - metadata = power_report_with_nested_metadata.gen_tag(tags) - - assert metadata == expected_metadata - check_report_metadata(original_metadata, power_report_with_nested_metadata) - - -def test_gen_tag_keep_all_the_report_metadata_with_all_tags(power_report_with_metadata): - tags = ['k1', 'k2', 'k3', 'k4'] - original_metadata = power_report_with_metadata.metadata - expected_metadata = get_expected_metadata_power_report_with_tags(power_report_with_metadata, tags) - - metadata = power_report_with_metadata.gen_tag(tags) - - assert metadata == expected_metadata - check_report_metadata(original_metadata, power_report_with_metadata) - - -def test_gen_tag_keep_some_report_metadata_with_some_tags(power_report_with_metadata): - tags = ['k1', 'k4'] - original_metadata = power_report_with_metadata.metadata - expected_metadata = get_expected_metadata_power_report_with_tags(power_report_with_metadata, tags) - - metadata = power_report_with_metadata.gen_tag(tags) - - assert metadata == expected_metadata - check_report_metadata(original_metadata, power_report_with_metadata) - - -def test_gen_tag_keep_all_the_report_nested_metadata_with_all_tags(power_report_with_nested_metadata): - tags = ['k1', 'k2', 'k3', 'k4'] - original_metadata = power_report_with_nested_metadata.metadata - expected_metadata = get_expected_metadata_power_report_with_tags(power_report_with_nested_metadata, tags) - - metadata = power_report_with_nested_metadata.gen_tag(tags) - - assert metadata == expected_metadata - check_report_metadata(original_metadata, power_report_with_nested_metadata) - - -def test_gen_tag_keep_some_report_nested_metadata_with_some_tags(power_report_with_nested_metadata): - tags = ['k1', 'k4'] - original_metadata = power_report_with_nested_metadata.metadata - expected_metadata = get_expected_metadata_power_report_with_tags(power_report_with_nested_metadata, tags) - - metadata = power_report_with_nested_metadata.gen_tag(tags) - - assert metadata == expected_metadata - check_report_metadata(original_metadata, power_report_with_nested_metadata) - - -def test_gen_tag_raise_exception_with_wrong_tags(power_report_with_metadata): - tags = ['kx', 'k4'] - original_metadata = power_report_with_metadata.metadata - - with pytest.raises(BadInputData): - _ = power_report_with_metadata.gen_tag(tags) - - check_report_metadata(original_metadata, power_report_with_metadata) - - -def test_gen_tag_raise_exception_with_wrong_tags_and_nested_metadata(power_report_with_nested_metadata): - tags = ['k1', 'k4_k2_k1'] - original_metadata = power_report_with_nested_metadata.metadata - - with pytest.raises(BadInputData): - _ = power_report_with_nested_metadata.gen_tag(tags) - - check_report_metadata(original_metadata, power_report_with_nested_metadata) - - -def test_to_influxdb_doesnt_add_extra_metadata_for_power_report_with_empty_metadata_and_empty_tag_list( - power_report_without_metadata): - tags = [] - expected_influxdb_document = get_expected_influxdb_document(power_report_without_metadata, tags) - - influxdb_document = PowerReport.to_influxdb(power_report_without_metadata, tags) - - assert influxdb_document == expected_influxdb_document - - -def test_to_influxdb_doesnt_add_extra_metadata_for_power_report_with_empty_metadata_and_without_tags( - power_report_without_metadata): - tags = None - expected_influxdb_document = get_expected_influxdb_document(power_report_without_metadata, tags) - - influxdb_document = PowerReport.to_influxdb(power_report_without_metadata, tags) - - assert influxdb_document == expected_influxdb_document - - -def test_to_influxdb_add_all_metadata_for_power_report_with_metadata_and_empty_tag_list( - power_report_with_metadata): - tags = [] - expected_influxdb_document = get_expected_influxdb_document(power_report_with_metadata, tags) - +@pytest.mark.parametrize('tags', [None, [], ['scope', 'socket', 'formula']]) +def test_to_influxdb_return_all_metadata_as_tags_for_report_with_metadata(tags, power_report_with_metadata, power_report_with_metadata_expected_tags): + """ + Test to serialize a report (with single-level metadata) using a tags selector that should return all tags for the InfluxDB database. + """ influxdb_document = PowerReport.to_influxdb(power_report_with_metadata, tags) - assert influxdb_document == expected_influxdb_document - - -def test_to_influxdb_add_all_metadata_for_power_report_with_metadata_and_without_tags( - power_report_with_metadata): - tags = None - expected_influxdb_document = get_expected_influxdb_document(power_report_with_metadata, tags) - - influxdb_document = PowerReport.to_influxdb(power_report_with_metadata, tags) + assert set(influxdb_document['tags']) == power_report_with_metadata_expected_tags - assert influxdb_document == expected_influxdb_document + assert influxdb_document['tags']['scope'] == power_report_with_metadata.metadata['scope'] + assert influxdb_document['tags']['socket'] == power_report_with_metadata.metadata['socket'] + assert influxdb_document['tags']['formula'] == power_report_with_metadata.metadata['formula'] + assert influxdb_document['tags']['sensor'] == power_report_with_metadata.sensor + assert influxdb_document['tags']['target'] == power_report_with_metadata.target -def test_to_influxdb_add_all_metadata_for_power_report_with_metadata_and_all_tags( - power_report_with_metadata): - tags = ['k1', 'k2', 'k3', 'k4'] - expected_influxdb_document = get_expected_influxdb_document(power_report_with_metadata, tags) +def test_to_influxdb_return_subset_metadata_as_tags_for_report_with_metadata(power_report_with_metadata): + """ + Test to serialize a report (with single-level metadata) with a subset of its tags for the InfluxDB database. + """ + tags = ['scope'] influxdb_document = PowerReport.to_influxdb(power_report_with_metadata, tags) - assert influxdb_document == expected_influxdb_document - - -def test_to_influxdb_add_all_metadata_for_power_report_with_nested_metadata_and_all_tags( - power_report_with_nested_metadata): - tags = ['k1', 'k2', 'k3', 'k4'] - expected_influxdb_document = get_expected_influxdb_document(power_report_with_nested_metadata, tags) - - influxdb_document = PowerReport.to_influxdb(power_report_with_nested_metadata, tags) - - assert influxdb_document == expected_influxdb_document - + assert set(influxdb_document['tags']) == set(tags) | {'sensor', 'target'} -def test_to_influxdb_add_some_metadata_for_power_report_with_metadata_and_some_tags( - power_report_with_metadata): - tags = ['k1', 'k2', 'k4'] - expected_influxdb_document = get_expected_influxdb_document(power_report_with_metadata, tags) + assert influxdb_document['tags']['scope'] == power_report_with_metadata.metadata['scope'] - influxdb_document = PowerReport.to_influxdb(power_report_with_metadata, tags) - - assert influxdb_document == expected_influxdb_document + assert influxdb_document['tags']['sensor'] == power_report_with_metadata.sensor + assert influxdb_document['tags']['target'] == power_report_with_metadata.target -def test_to_influxdb_add_some_metadata_for_power_report_with_nested_metadata_and_some_tags( - power_report_with_nested_metadata): - tags = ['k1', 'k3', 'k4'] - expected_influxdb_document = get_expected_influxdb_document(power_report_with_nested_metadata, tags) - +@pytest.mark.parametrize('tags', [None, [], ['scope', 'socket', 'formula', 'k8s_app_kubernetes_io_name', 'k8s_app_kubernetes_io_managed_by', 'k8s_helm_sh_chart']]) +def test_to_influxdb_return_all_metadata_as_tags_for_report_with_nested_metadata(tags, power_report_with_nested_metadata, power_report_with_nested_metadata_expected_tags): + """ + Test to serialize a report (with nested metadata) using a tags selector that should return all tags for the InfluxDB database. + """ influxdb_document = PowerReport.to_influxdb(power_report_with_nested_metadata, tags) - assert influxdb_document == expected_influxdb_document - - -def test_to_influxdb_raise_exception_for_power_report_with_metadata_and_some_tags( - power_report_with_metadata): - tags = ['k8888', 'k2', 'k4'] - - with pytest.raises(BadInputData): - _ = PowerReport.to_influxdb(power_report_with_metadata, tags) - - -def test_to_influxdb_raise_exception_with_wrong_tags_and_nested_metadata( - power_report_with_nested_metadata): - tags = ['k1', 'k4_k1', 'k333'] - - with pytest.raises(BadInputData): - _ = PowerReport.to_influxdb(power_report_with_nested_metadata, tags) + assert set(influxdb_document['tags']) == power_report_with_nested_metadata_expected_tags + assert influxdb_document['tags']['scope'] == power_report_with_nested_metadata.metadata['scope'] + assert influxdb_document['tags']['socket'] == power_report_with_nested_metadata.metadata['socket'] + assert influxdb_document['tags']['formula'] == power_report_with_nested_metadata.metadata['formula'] + assert influxdb_document['tags']['k8s_app_kubernetes_io_name'] == power_report_with_nested_metadata.metadata['k8s']['app.kubernetes.io/name'] + assert influxdb_document['tags']['k8s_app_kubernetes_io_managed_by'] == power_report_with_nested_metadata.metadata['k8s']['app.kubernetes.io/managed-by'] + assert influxdb_document['tags']['k8s_helm_sh_chart'] == power_report_with_nested_metadata.metadata['k8s']['helm.sh/chart'] -def test_to_prometheus_doesnt_add_extra_metadata_for_power_report_with_empty_metadata_and_empty_tag_list( - power_report_without_metadata): - tags = [] - expected_prometheus_document = get_expected_prometheus_document(power_report_without_metadata, tags) + assert influxdb_document['tags']['sensor'] == power_report_with_nested_metadata.sensor + assert influxdb_document['tags']['target'] == power_report_with_nested_metadata.target - prometheus_document = PowerReport.to_prometheus(power_report_without_metadata, tags) - assert prometheus_document == expected_prometheus_document - - -def test_to_prometheus_doesnt_add_extra_metadata_for_power_report_with_empty_metadata_and_without_tags( - power_report_without_metadata): - tags = None - expected_prometheus_document = get_expected_prometheus_document(power_report_without_metadata, tags) +def test_to_influxdb_return_subset_metadata_as_tags_for_report_with_nested_metadata(power_report_with_nested_metadata): + """ + Test to serialize a report (with nested metadata) with a subset of its tags for the InfluxDB database. + """ + tags = ['scope', 'socket', 'k8s_app_kubernetes_io_name', 'k8s_helm_sh_chart'] + influxdb_document = PowerReport.to_influxdb(power_report_with_nested_metadata, tags) - prometheus_document = PowerReport.to_prometheus(power_report_without_metadata, tags) + assert set(influxdb_document['tags']) == set(tags) | {'sensor', 'target'} - assert prometheus_document == expected_prometheus_document + assert influxdb_document['tags']['scope'] == power_report_with_nested_metadata.metadata['scope'] + assert influxdb_document['tags']['socket'] == power_report_with_nested_metadata.metadata['socket'] + assert influxdb_document['tags']['k8s_app_kubernetes_io_name'] == power_report_with_nested_metadata.metadata['k8s']['app.kubernetes.io/name'] + assert influxdb_document['tags']['k8s_helm_sh_chart'] == power_report_with_nested_metadata.metadata['k8s']['helm.sh/chart'] + assert influxdb_document['tags']['sensor'] == power_report_with_nested_metadata.sensor + assert influxdb_document['tags']['target'] == power_report_with_nested_metadata.target -def test_to_prometheus_add_all_metadata_for_power_report_with_metadata_and_empty_tag_list( - power_report_with_metadata): - tags = [] - expected_prometheus_document = get_expected_prometheus_document(power_report_with_metadata, tags) +@pytest.mark.parametrize('tags', [None, [], ['scope', 'socket', 'formula']]) +def test_to_prometheus_return_all_metadata_as_tags_for_report_with_metadata(tags, power_report_with_metadata, power_report_with_metadata_expected_tags): + """ + Test to serialize a report (with single-level metadata) using a tags selector that should return all tags for the Prometheus database. + """ prometheus_document = PowerReport.to_prometheus(power_report_with_metadata, tags) - assert prometheus_document == expected_prometheus_document - + assert set(prometheus_document['tags']) == power_report_with_metadata_expected_tags -def test_to_prometheus_add_all_metadata_for_power_report_with_metadata_and_without_tags( - power_report_with_metadata): - tags = None - expected_prometheus_document = get_expected_prometheus_document(power_report_with_metadata, tags) + assert prometheus_document['tags']['scope'] == power_report_with_metadata.metadata['scope'] + assert prometheus_document['tags']['socket'] == power_report_with_metadata.metadata['socket'] + assert prometheus_document['tags']['formula'] == power_report_with_metadata.metadata['formula'] - prometheus_document = PowerReport.to_prometheus(power_report_with_metadata, tags) - - assert prometheus_document == expected_prometheus_document + assert prometheus_document['tags']['sensor'] == power_report_with_metadata.sensor + assert prometheus_document['tags']['target'] == power_report_with_metadata.target -def test_to_prometheus_add_all_metadata_for_power_report_with_metadata_and_all_tags( - power_report_with_metadata): - tags = ['k2', 'k3', 'k1', 'k4'] - expected_prometheus_document = get_expected_prometheus_document(power_report_with_metadata, tags) - +def test_to_prometheus_return_subset_metadata_as_tags_for_report_with_metadata(power_report_with_metadata): + """ + Test to serialize a report (with single-level metadata) with a subset of its tags for the Prometheus database. + """ + tags = ['formula'] prometheus_document = PowerReport.to_prometheus(power_report_with_metadata, tags) - assert prometheus_document == expected_prometheus_document - + assert set(prometheus_document['tags']) == set(tags) | {'sensor', 'target'} -def test_to_prometheus_add_all_metadata_for_power_report_with_nested_metadata_and_all_tags( - power_report_with_nested_metadata): - tags = ['k1', 'k2', 'k3', 'k4'] - expected_prometheus_document = get_expected_prometheus_document(power_report_with_nested_metadata, tags) - - prometheus_document = PowerReport.to_prometheus(power_report_with_nested_metadata, tags) + assert prometheus_document['tags']['formula'] == power_report_with_metadata.metadata['formula'] - assert prometheus_document == expected_prometheus_document + assert prometheus_document['tags']['sensor'] == power_report_with_metadata.sensor + assert prometheus_document['tags']['target'] == power_report_with_metadata.target -def test_to_prometheus_add_some_metadata_for_power_report_with_metadata_and_some_tags( - power_report_with_metadata): - tags = ['k4', 'k3', 'k1'] - expected_prometheus_document = get_expected_prometheus_document(power_report_with_metadata, tags) +@pytest.mark.parametrize('tags', [None, [], ['scope', 'socket', 'formula', 'k8s_app_kubernetes_io_name', 'k8s_app_kubernetes_io_managed_by', 'k8s_helm_sh_chart']]) +def test_to_prometheus_return_all_metadata_as_tags_for_report_with_nested_metadata(tags, power_report_with_nested_metadata, power_report_with_nested_metadata_expected_tags): + """ + Test to serialize a report (with nested metadata) using a tags selector that should return all tags for the Prometheus database. + """ + prometheus_document = PowerReport.to_prometheus(power_report_with_nested_metadata, tags) - prometheus_document = PowerReport.to_prometheus(power_report_with_metadata, tags) + assert set(prometheus_document['tags']) == power_report_with_nested_metadata_expected_tags - assert prometheus_document == expected_prometheus_document + assert prometheus_document['tags']['scope'] == power_report_with_nested_metadata.metadata['scope'] + assert prometheus_document['tags']['socket'] == power_report_with_nested_metadata.metadata['socket'] + assert prometheus_document['tags']['formula'] == power_report_with_nested_metadata.metadata['formula'] + assert prometheus_document['tags']['k8s_app_kubernetes_io_name'] == power_report_with_nested_metadata.metadata['k8s']['app.kubernetes.io/name'] + assert prometheus_document['tags']['k8s_app_kubernetes_io_managed_by'] == power_report_with_nested_metadata.metadata['k8s']['app.kubernetes.io/managed-by'] + assert prometheus_document['tags']['k8s_helm_sh_chart'] == power_report_with_nested_metadata.metadata['k8s']['helm.sh/chart'] + assert prometheus_document['tags']['sensor'] == power_report_with_nested_metadata.sensor + assert prometheus_document['tags']['target'] == power_report_with_nested_metadata.target -def test_to_prometheus_add_some_metadata_for_power_report_with_nested_metadata_and_some_tags( - power_report_with_nested_metadata): - tags = ['k1', 'k2', 'k4'] - expected_prometheus_document = get_expected_prometheus_document(power_report_with_nested_metadata, tags) +def test_to_prometheus_return_subset_metadata_as_tags_for_report_with_nested_metadata(power_report_with_nested_metadata): + """ + Test to serialize a report (with nested metadata) with a subset of its tags for the Prometheus database. + """ + tags = ['scope', 'formula', 'k8s_app_kubernetes_io_name', 'k8s_app_kubernetes_io_managed_by'] prometheus_document = PowerReport.to_prometheus(power_report_with_nested_metadata, tags) - assert prometheus_document == expected_prometheus_document - - -def test_to_prometheus_raise_exception_for_power_report_with_metadata_and_some_tags( - power_report_with_metadata): - tags = ['k888', 'k2', 'k4'] - - with pytest.raises(BadInputData): - _ = PowerReport.to_prometheus(power_report_with_metadata, tags) + assert set(prometheus_document['tags']) == set(tags) | {'sensor', 'target'} + assert prometheus_document['tags']['scope'] == power_report_with_nested_metadata.metadata['scope'] + assert prometheus_document['tags']['formula'] == power_report_with_nested_metadata.metadata['formula'] + assert prometheus_document['tags']['k8s_app_kubernetes_io_name'] == power_report_with_nested_metadata.metadata['k8s']['app.kubernetes.io/name'] + assert prometheus_document['tags']['k8s_app_kubernetes_io_managed_by'] == power_report_with_nested_metadata.metadata['k8s']['app.kubernetes.io/managed-by'] -def test_to_prometheus_raise_exception_with_wrong_tags_and_nested_metadata( - power_report_with_nested_metadata): - tags = ['k1', 'k4_k1', 'k333'] - - with pytest.raises(BadInputData): - _ = PowerReport.to_prometheus(power_report_with_nested_metadata, tags) + assert prometheus_document['tags']['sensor'] == power_report_with_nested_metadata.sensor + assert prometheus_document['tags']['target'] == power_report_with_nested_metadata.target diff --git a/tests/unit/report/test_report.py b/tests/unit/report/test_report.py index 49917eab..b785d735 100644 --- a/tests/unit/report/test_report.py +++ b/tests/unit/report/test_report.py @@ -1,21 +1,21 @@ # Copyright (c) 2021, INRIA # Copyright (c) 2021, University of Lille # All rights reserved. - +# # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: - +# # * Redistributions of source code must retain the above copyright notice, this # list of conditions and the following disclaimer. - +# # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. - +# # * Neither the name of the copyright holder nor the names of its # contributors may be used to endorse or promote products derived from # this software without specific prior written permission. - +# # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE @@ -27,30 +27,20 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import pytest - -from powerapi.report import Report from datetime import datetime - -@pytest.fixture() -def basic_report(): - return Report(timestamp=datetime.strptime('1970-09-01T09:09:10.543', "%Y-%m-%dT%H:%M:%S.%f"), sensor='toto', - target='all', metadata={"tag": 1}) - - -@pytest.fixture() -def expected_json_report(basic_report): - return {'timestamp': basic_report.timestamp, - 'sensor': basic_report.sensor, - 'target': basic_report.target, - 'metadata': basic_report.metadata} +from powerapi.report import Report def test_creating_report_with_metadata(): - report = Report(timestamp=datetime.strptime('1970-09-01T09:09:10.543', "%Y-%m-%dT%H:%M:%S.%f"), sensor='toto', - target='all', metadata={"tag": 1}) - assert report.metadata["tag"] == 1 + """ + Test creating a report with metadata. + """ + report = Report(datetime.now(), 'pytest', 'test', {'tag1': 1, 'tag2': {'2'}, 'tag3': '3'}) + + assert report.metadata["tag1"] == 1 + assert report.metadata["tag2"] == {'2'} + assert report.metadata["tag3"] == '3' def test_create_two_report_without_metadata_metadata_are_different(): @@ -65,18 +55,40 @@ def test_create_two_report_without_metadata_metadata_are_different(): assert a.metadata != b.metadata -def test_to_json(basic_report, expected_json_report): - - json = Report.to_json(report=basic_report) - assert 'sender_name' not in json - assert 'dispatcher_report_id' not in json - assert json == expected_json_report +def test_sanitize_tags_name(): + """ + Test sanitizing tag names from the metadata dictionary. + """ + tags = ['test-tag', 'app.kubernetes.io/name', 'helm.sh/chart'] + sanitized_tags = Report.sanitize_tags_name(tags) + assert len(sanitized_tags) == len(tags) + assert sanitized_tags['test-tag'] == 'test_tag' + assert sanitized_tags['app.kubernetes.io/name'] == 'app_kubernetes_io_name' + assert sanitized_tags['helm.sh/chart'] == 'helm_sh_chart' -def test_to_json_with_dispatcher_report_id(basic_report, expected_json_report): - basic_report.dispatcher_report_id = 10 - json = Report.to_json(report=basic_report) - assert 'sender_name' not in json - assert 'dispatcher_report_id' not in json - assert json == expected_json_report +def test_flatten_metadata_dict(): + """ + Test flattening a report metadata dictionary. + """ + report_metadata = { + 'scope': 'cpu', + 'socket': 0, + 'formula': '0000000000000000000000000000000000000000', + 'k8s': { + 'app.kubernetes.io/name': 'test', + 'app.kubernetes.io/instance': 'test-abcxyz', + 'app.kubernetes.io/managed-by': 'pytest', + 'helm.sh/chart': 'powerapi-pytest-1.0.0' + } + } + flattened_metadata = Report.flatten_tags(report_metadata, '/') + + assert flattened_metadata['scope'] == 'cpu' + assert flattened_metadata['socket'] == 0 + assert flattened_metadata['formula'] == '0000000000000000000000000000000000000000' + assert flattened_metadata['k8s/app.kubernetes.io/name'] == 'test' + assert flattened_metadata['k8s/app.kubernetes.io/instance'] == 'test-abcxyz' + assert flattened_metadata['k8s/app.kubernetes.io/managed-by'] == 'pytest' + assert flattened_metadata['k8s/helm.sh/chart'] == 'powerapi-pytest-1.0.0'