Skip to content

Commit

Permalink
gcp_cost: only query current month
Browse files Browse the repository at this point in the history
Previously also the costs of the previous month were queried, and shown
in the details. With this change it is no longer shown in details,
please refer to the newly introduced metric if previous cost development
is important for you.

This should reduce the cost of querying the cost significantly.

Change-Id: I6e92b3e305f8236a338956f2c2db1120dc90bd5f
  • Loading branch information
BenediktSeidl committed Jan 20, 2025
1 parent d47ce87 commit c780e75
Show file tree
Hide file tree
Showing 3 changed files with 14 additions and 39 deletions.
39 changes: 11 additions & 28 deletions cmk/plugins/gcp/agent_based/gcp_cost.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
import json
from collections.abc import Mapping
from dataclasses import dataclass
from itertools import groupby
from typing import Any

from pydantic import BaseModel, ConfigDict, field_validator
Expand All @@ -18,9 +17,7 @@
CheckPlugin,
CheckResult,
DiscoveryResult,
Result,
Service,
State,
StringTable,
)

Expand Down Expand Up @@ -51,37 +48,26 @@ def date(self) -> str:
class ProjectCost:
project: str
current_month: Cost
previous_month: Cost | None


Section = Mapping[ProjectId, ProjectCost]


def keyfunc(x: Mapping[str, str]) -> str:
return x["id"]


def parse(string_table: StringTable) -> Section:
query_month = datetime.datetime.strptime(json.loads(string_table[0][0])["query_month"], "%Y%m")
all_rows = sorted([json.loads(line[0]) for line in string_table[1:]], key=keyfunc)

section = {}
for project_id, rows in groupby(all_rows, key=keyfunc):
month_costs = sorted(
[Cost.model_validate(r) for r in rows], key=lambda c: c.month, reverse=True
)
if len(month_costs) > 1:
cost = ProjectCost(
current_month=month_costs[0],
previous_month=month_costs[1],
project=month_costs[1].project,
)
else:
cost = ProjectCost(
current_month=month_costs[0], previous_month=None, project=month_costs[0].project
)
if cost.current_month.month != query_month:
for row in [json.loads(line[0]) for line in string_table[1:]]:
cost = Cost.model_validate(row)
if cost.month != query_month:
# just to make sure, query should only return values of that month
continue
section[project_id] = cost

project = ProjectCost(
cost.project,
current_month=cost,
)
section[row["id"]] = project

return section

Expand All @@ -108,9 +94,6 @@ def check(item: str, params: Mapping[str, Any], section: Section) -> CheckResult
label=current_month.date(),
)

if previous_month := project_costs.previous_month:
yield Result(state=State.OK, notice=previous_month.to_details())


check_plugin_gcp_cost = CheckPlugin(
name="gcp_cost",
Expand Down
8 changes: 2 additions & 6 deletions cmk/plugins/gcp/special_agents/agent_gcp.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,6 @@ def list_assets(self, request: Any) -> Iterable[asset_v1.Asset]:

def list_costs(self, tableid: str) -> tuple[Schema, Pages]:
first_of_month = self.date.replace(day=1)
first_of_previous_month = (first_of_month - datetime.timedelta(days=1)).replace(day=1)
if "`" in tableid:
raise ValueError("tableid contains invalid character")

Expand All @@ -117,11 +116,8 @@ def list_costs(self, tableid: str) -> tuple[Schema, Pages]:
"currency, "
"invoice.month "
f"FROM `{tableid}`"
'WHERE ('
f' invoice.month = "{first_of_month.strftime("%Y%m")}" '
f' OR invoice.month = "{first_of_previous_month.strftime("%Y%m")}" '
') '
f'AND DATE(_PARTITIONTIME) >= "{first_of_previous_month.strftime("%Y-%m-%d")}" '
f'WHERE invoice.month = "{first_of_month.strftime("%Y%m")}" '
f'AND DATE(_PARTITIONTIME) >= "{first_of_month.strftime("%Y-%m-%d")}" '
"AND project.name IS NOT NULL "
"GROUP BY project.name, project.id, currency, invoice.month "
"ORDER BY project.name, invoice.month"
Expand Down
6 changes: 1 addition & 5 deletions tests/unit/cmk/plugins/gcp/agent_based/test_gcp_cost.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,6 @@ def test_gcp_cost_check(section: Section, item: str) -> None:
assert results == [
Result(state=State.OK, summary="July 2022: 42.21 EUR"),
Metric("gcp_cost_per_month", 42.21),
Result(state=State.OK, notice="June 2022: 1337.00 EUR"),
]


Expand Down Expand Up @@ -126,7 +125,4 @@ def test_gcp_cost_check_levels(
result: list[Result | Metric], params: Mapping[str, Any], section: Section
) -> None:
results = list(check(item="test1", params=params, section=section))
assert results == [
*result,
Result(state=State.OK, notice="June 2022: 1337.00 EUR"),
]
assert results == result

0 comments on commit c780e75

Please sign in to comment.