Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor/issue209/prometheus consolidation #210

Merged
merged 11 commits into from
Nov 22, 2023
Merged

Refactor/issue209/prometheus consolidation #210

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
bd18d81
refactor: Remove unncessary code
roda82 Nov 14, 2023
019d49e
refactor: Adapt PrometheusDB for provising a unique and simple implem…
roda82 Nov 14, 2023
4ab9add
refactor: Clean code
roda82 Nov 15, 2023
e9d21c7
refactor: Adapt PrometheusDB to use all metadata keys of the first re…
roda82 Nov 17, 2023
3fdce02
refactor: Adapt K8sPreProcessor to add empty metadata values if infor…
roda82 Nov 17, 2023
8ca9c51
refactor: Adapt Pusher ReportHandler to ignore report saving when rep…
roda82 Nov 17, 2023
8c61ef4
test: Add test case for PrometheusDB regarding labels
roda82 Nov 17, 2023
eda819c
fix: Correct bug related to arguments default values not added to con…
roda82 Nov 22, 2023
fe186ca
refactor: Add 127.0.0.1 as default uri value for PrometheusDB
roda82 Nov 22, 2023
5501b9f
refactor: Rename variable name to avoid default string
roda82 Nov 22, 2023
e60f2f6
refactor: Clean code
roda82 Nov 22, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 20 additions & 50 deletions powerapi/cli/common_cli_parsing_manager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@
from powerapi.cli.parsing_manager import RootConfigParsingManager, SubgroupConfigParsingManager
from powerapi.cli.config_parser import store_true
from powerapi.cli.config_parser import MissingValueException
from powerapi.database.prometheus_db import DEFAULT_METRIC_DESCRIPTION, DEFAULT_MODEL_VALUE, DEFAULT_PUSHER_NAME, \

Check warning on line 35 in powerapi/cli/common_cli_parsing_manager.py

View check run for this annotation

Codecov / codecov/patch

powerapi/cli/common_cli_parsing_manager.py#L35 

Added line #L35 was not covered by tests
DEFAULT_ADDRESS
from powerapi.exception import BadTypeException, BadContextException, UnknownArgException

POWERAPI_ENVIRONMENT_VARIABLE_PREFIX = 'POWERAPI_'
Expand All @@ -40,6 +42,8 @@
POWERAPI_PRE_PROCESSOR_ENVIRONMENT_VARIABLE_PREFIX = POWERAPI_ENVIRONMENT_VARIABLE_PREFIX + 'PRE_PROCESSOR_'
POWERAPI_POST_PROCESSOR_ENVIRONMENT_VARIABLE_PREFIX = POWERAPI_ENVIRONMENT_VARIABLE_PREFIX + 'POST_PROCESSOR'

TAGS_ARGUMENT_HELP_TEXT = 'specify report tags'

Check warning on line 45 in powerapi/cli/common_cli_parsing_manager.py

View check run for this annotation

Codecov / codecov/patch

powerapi/cli/common_cli_parsing_manager.py#L45 

Added line #L45 was not covered by tests


def extract_file_names(arg, val, args, acc):
"""
Expand Down Expand Up @@ -250,69 +254,35 @@
subgroup_parser=subparser_mongo_output
)

subparser_prom_output = SubgroupConfigParsingManager("prom")
subparser_prom_output.add_argument("t", "tags", help_text="specify report tags")
subparser_prom_output.add_argument("u", "uri", help_text="specify server uri")
subparser_prom_output.add_argument(
subparser_prometheus_output = SubgroupConfigParsingManager("prometheus")
subparser_prometheus_output.add_argument("t", "tags", help_text=TAGS_ARGUMENT_HELP_TEXT)
subparser_prometheus_output.add_argument("u", "uri", help_text="specify server uri",

Check warning on line 259 in powerapi/cli/common_cli_parsing_manager.py

View check run for this annotation

Codecov / codecov/patch

powerapi/cli/common_cli_parsing_manager.py#L257-L259 

Added lines #L257 - L259 were not covered by tests
default_value=DEFAULT_ADDRESS)
subparser_prometheus_output.add_argument(

Check warning on line 261 in powerapi/cli/common_cli_parsing_manager.py

View check run for this annotation

Codecov / codecov/patch

powerapi/cli/common_cli_parsing_manager.py#L261 

Added line #L261 was not covered by tests
"p", "port", help_text="specify server port", argument_type=int
)
subparser_prom_output.add_argument(
subparser_prometheus_output.add_argument(

Check warning on line 264 in powerapi/cli/common_cli_parsing_manager.py

View check run for this annotation

Codecov / codecov/patch

powerapi/cli/common_cli_parsing_manager.py#L264 

Added line #L264 was not covered by tests
"M", "metric_name", help_text="specify metric name"
)
subparser_prom_output.add_argument(
subparser_prometheus_output.add_argument(

Check warning on line 267 in powerapi/cli/common_cli_parsing_manager.py

View check run for this annotation

Codecov / codecov/patch

powerapi/cli/common_cli_parsing_manager.py#L267 

Added line #L267 was not covered by tests
"d",
"metric_description",
help_text="specify metric description",
default_value="energy consumption",
)
help_text = "specify number of second for the value must be aggregated before compute statistics on them"
subparser_prom_output.add_argument(
"A", "aggregation_period", help_text=help_text, default_value=15, argument_type=int
)

subparser_prom_output.add_argument(
"m",
"model",
help_text="specify data type that will be stored in the database",
default_value="PowerReport",
)
subparser_prom_output.add_argument(
"n", "name", help_text="specify pusher name", default_value="pusher_prom"
)
self.add_subgroup_parser(
subgroup_name="output",
subgroup_parser=subparser_prom_output
default_value=DEFAULT_METRIC_DESCRIPTION
)

subparser_direct_prom_output = SubgroupConfigParsingManager("direct_prom")
subparser_direct_prom_output.add_argument(
"t", "tags", help_text="specify report tags"
)
subparser_direct_prom_output.add_argument("a", "uri", help_text="specify server uri")
subparser_direct_prom_output.add_argument(
"p", "port", help_text="specify server port", argument_type=int
)
subparser_direct_prom_output.add_argument(
"M", "metric_name", help_text="specify metric name"
)
subparser_direct_prom_output.add_argument(
"d",
"metric_description",
help_text="specify metric description",
default_value="energy consumption",
)
subparser_direct_prom_output.add_argument(
subparser_prometheus_output.add_argument(

Check warning on line 274 in powerapi/cli/common_cli_parsing_manager.py

View check run for this annotation

Codecov / codecov/patch

powerapi/cli/common_cli_parsing_manager.py#L274 

Added line #L274 was not covered by tests
"m",
"model",
help_text="specify data type that will be stored in the database",
default_value="PowerReport",
default_value=DEFAULT_MODEL_VALUE,
)
subparser_direct_prom_output.add_argument(
"n", "name", help_text="specify pusher name", default_value="pusher_prom"
subparser_prometheus_output.add_argument(

Check warning on line 280 in powerapi/cli/common_cli_parsing_manager.py

View check run for this annotation

Codecov / codecov/patch

powerapi/cli/common_cli_parsing_manager.py#L280 

Added line #L280 was not covered by tests
"n", "name", help_text="specify pusher name", default_value=DEFAULT_PUSHER_NAME
)
self.add_subgroup_parser(
subgroup_name="output",
subgroup_parser=subparser_direct_prom_output
subgroup_parser=subparser_prometheus_output
)

subparser_csv_output = SubgroupConfigParsingManager("csv")
Expand All @@ -328,7 +298,7 @@
default_value="PowerReport",
)

subparser_csv_output.add_argument("t", "tags", help_text="specify report tags")
subparser_csv_output.add_argument("t", "tags", help_text=TAGS_ARGUMENT_HELP_TEXT)

Check warning on line 301 in powerapi/cli/common_cli_parsing_manager.py

View check run for this annotation

Codecov / codecov/patch

powerapi/cli/common_cli_parsing_manager.py#L301 

Added line #L301 was not covered by tests
subparser_csv_output.add_argument(
"n", "name", help_text="specify pusher name", default_value="pusher_csv"
)
Expand All @@ -339,7 +309,7 @@

subparser_influx_output = SubgroupConfigParsingManager("influxdb")
subparser_influx_output.add_argument("u", "uri", help_text="specify InfluxDB uri")
subparser_influx_output.add_argument("t", "tags", help_text="specify report tags")
subparser_influx_output.add_argument("t", "tags", help_text=TAGS_ARGUMENT_HELP_TEXT)

Check warning on line 312 in powerapi/cli/common_cli_parsing_manager.py

View check run for this annotation

Codecov / codecov/patch

powerapi/cli/common_cli_parsing_manager.py#L312 

Added line #L312 was not covered by tests
subparser_influx_output.add_argument(
"d", "db", help_text="specify InfluxDB database name"
)
Expand Down Expand Up @@ -385,7 +355,7 @@

subparser_influx2_output = SubgroupConfigParsingManager("influxdb2")
subparser_influx2_output.add_argument("u", "uri", help_text="specify InfluxDB uri")
subparser_influx2_output.add_argument("t", "tags", help_text="specify report tags")
subparser_influx2_output.add_argument("t", "tags", help_text=TAGS_ARGUMENT_HELP_TEXT)

Check warning on line 358 in powerapi/cli/common_cli_parsing_manager.py

View check run for this annotation

Codecov / codecov/patch

powerapi/cli/common_cli_parsing_manager.py#L358 

Added line #L358 was not covered by tests
subparser_influx2_output.add_argument("k", "token",
help_text="specify token for accessing the database")
subparser_influx2_output.add_argument("g", "org",
Expand Down
2 changes: 1 addition & 1 deletion powerapi/cli/config_parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -259,7 +259,7 @@ def validate(self, conf: dict) -> dict:
elif current_argument_name != 'type':
raise UnknownArgException(argument_name=current_argument_name)

return conf
return self.normalize_configuration(conf=conf)

def normalize_configuration(self, conf: dict) -> dict:
"""
Expand Down
20 changes: 8 additions & 12 deletions powerapi/cli/generator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
TIMEOUT_QUERY_DEFAULT_VALUE
from powerapi.processor.pre.libvirt.libvirt_pre_processor_actor import LibvirtPreProcessorActor
from powerapi.report import HWPCReport, PowerReport, ControlReport, ProcfsReport, Report, FormulaReport
from powerapi.database import MongoDB, CsvDB, InfluxDB, OpenTSDB, SocketDB, PrometheusDB, DirectPrometheusDB, \
from powerapi.database import MongoDB, CsvDB, InfluxDB, OpenTSDB, SocketDB, PrometheusDB, \
VirtioFSDB, FileDB
from powerapi.puller import PullerActor
from powerapi.pusher import PusherActor
Expand Down Expand Up @@ -181,7 +181,7 @@ def __init__(self, component_group_name: str):
self.db_factory = {
'mongodb': lambda db_config: MongoDB(report_type=db_config['model'], uri=db_config['uri'],
db_name=db_config['db'], collection_name=db_config['collection']),
'socket': lambda db_config: SocketDB(db_config['model'], db_config['port']),
'socket': lambda db_config: SocketDB(report_type=db_config['model'], port=db_config['port']),
'csv': lambda db_config: CsvDB(report_type=db_config['model'], tags=gen_tag_list(db_config),
current_path=os.getcwd() if 'directory' not in db_config else db_config[
'directory'],
Expand All @@ -196,16 +196,12 @@ def __init__(self, component_group_name: str):
port=None if 'port' not in db_config else db_config['port']),
'opentsdb': lambda db_config: OpenTSDB(report_type=db_config['model'], host=db_config['uri'],
port=db_config['port'], metric_name=db_config['metric_name']),
'prom': lambda db_config: PrometheusDB(report_type=db_config['model'], port=db_config['port'],
address=db_config['uri'], metric_name=db_config['metric_name'],
metric_description=db_config['metric_description'],
aggregation_periode=db_config['aggregation_period'],
tags=gen_tag_list(db_config)),
'direct_prom': lambda db_config: DirectPrometheusDB(report_type=db_config['model'], port=db_config['port'],
address=db_config['uri'],
metric_name=db_config['metric_name'],
metric_description=db_config['metric_description'],
tags=gen_tag_list(db_config)),
'prometheus': lambda db_config: PrometheusDB(report_type=db_config['model'],
port=db_config['port'],
address=db_config['uri'],
metric_name=db_config['metric_name'],
metric_description=db_config['metric_description'],
tags=gen_tag_list(db_config)),
'virtiofs': lambda db_config: VirtioFSDB(report_type=db_config['model'],
vm_name_regexp=db_config['vm_name_regexp'],
root_directory_name=db_config['root_directory_name'],
Expand Down
1 change: 0 additions & 1 deletion powerapi/database/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,5 @@
from powerapi.database.influxdb2 import InfluxDB2
from powerapi.database.prometheus_db import PrometheusDB
from powerapi.database.virtiofs_db import VirtioFSDB
from powerapi.database.direct_prometheus_db import DirectPrometheusDB
from powerapi.database.socket_db import SocketDB
from powerapi.database.file_db import FileDB
112 changes: 0 additions & 112 deletions powerapi/database/direct_prometheus_db.py
View file
Open in desktop

This file was deleted.

Loading