diff --git a/diworker/diworker/importers/aws.py b/diworker/diworker/importers/aws.py index 045f33e5..08fbf33d 100644 --- a/diworker/diworker/importers/aws.py +++ b/diworker/diworker/importers/aws.py @@ -2,6 +2,7 @@ import csv import gzip import logging +import math import os import re import pyarrow @@ -477,6 +478,8 @@ def load_parquet_report(self, report_path, account_id_ca_id_map, elif field_name == 'lineItem/UsageType': if 'BoxUsage' in value: chunk[expense_num]['box_usage'] = True + if isinstance(value, float) and math.isnan(value): + value = 0 if value: chunk[expense_num][field_name] = value diff --git a/diworker/diworker/migrations/2024091713320000_aws_raw_expenses_nan_to_zero.py b/diworker/diworker/migrations/2024091713320000_aws_raw_expenses_nan_to_zero.py new file mode 100644 index 00000000..d8fc2845 --- /dev/null +++ b/diworker/diworker/migrations/2024091713320000_aws_raw_expenses_nan_to_zero.py @@ -0,0 +1,99 @@ +import logging +import math +from pymongo import UpdateOne +from optscale_client.rest_api_client.client_v2 import Client as RestClient +from diworker.diworker.migrations.base import BaseMigration +""" +Update NaN values to 0 in raw expenses for AWS +""" +CHUNK_SIZE = 200 +LOG = logging.getLogger(__name__) + +EXPENSE_FIELDS = [ + 'cost', + 'discount_bundled_discount', + 'discount_total_discount', + 'lineItem/NormalizationFactor', + 'lineItem/BlendedCost', + 'lineItem/NetUnblendedCost', + 'lineItem/UnblendedCost', + 'lineItem/UsageAmount', + 'reservation/EffectiveCost', + 'reservation/NetEffectiveCost', + 'reservation/TotalReservedNormalizedUnits', + 'reservation/TotalReservedUnits', + 'reservation/AmortizedUpfrontCostForUsage', + 'reservation/NetAmortizedUpfrontCostForUsage', + 'reservation/AmortizedUpfrontFeeForBillingPeriod', + 'reservation/NetAmortizedUpfrontFeeForBillingPeriod', + 'reservation/RecurringFeeForUsage', + 'reservation/NetRecurringFeeForUsage', + 'reservation/UnusedRecurringFee', + 'reservation/NetUnusedRecurringFee', + 'reservation/UpfrontValue', + 'reservation/NetUpfrontValue', + 'reservation/UnusedAmortizedUpfrontFeeForBillingPeriod', + 'reservation/NetUnusedAmortizedUpfrontFeeForBillingPeriod', + 'savingsPlan/AmortizedUpfrontCommitmentForBillingPeriod', + 'savingsPlan/NetAmortizedUpfrontCommitmentForBillingPeriod', + 'savingsPlan/RecurringCommitmentForBillingPeriod', + 'savingsPlan/NetRecurringCommitmentForBillingPeriod', + 'savingsPlan/SavingsPlanEffectiveCost', + 'savingsPlan/NetSavingsPlanEffectiveCost', + 'savingsPlan/SavingsPlanRate', + 'savingsPlan/TotalCommitmentToDate', + 'pricing/publicOnDemandCost', +] + + +class Migration(BaseMigration): + @property + def mongo_raw(self): + return self.db.raw_expenses + + @property + def rest_cl(self): + if self._rest_cl is None: + self._rest_cl = RestClient( + url=self.config_cl.restapi_url(), + secret=self.config_cl.cluster_secret()) + return self._rest_cl + + def get_cloud_accs(self): + cloud_accounts_ids = set() + _, organizations = self.rest_cl.organization_list({ + 'with_connected_accounts': True, 'is_demo': False}) + for org in organizations['organizations']: + _, accounts = self.rest_cl.cloud_account_list( + org['id'], type='aws_cnr') + for cloud_account in accounts['cloud_accounts']: + if cloud_account['auto_import']: + cloud_accounts_ids.add(cloud_account['id']) + return cloud_accounts_ids + + def upgrade(self): + cloud_accs = self.get_cloud_accs() + for i, cloud_acc_id in enumerate(cloud_accs): + LOG.info('Starting processing for cloud account %s (%s/%s)' % ( + cloud_acc_id, i+1, len(cloud_accs))) + raw_expenses = self.mongo_raw.find({ + 'cloud_account_id': cloud_acc_id, + 'bill/BillingPeriodStartDate': {'$exists': True}, + '$or': [{x: math.nan} for x in EXPENSE_FIELDS] + }) + chunk = [] + for expense in raw_expenses: + fields = [x for x, v in expense.items() + if isinstance(v, float) and math.isnan(v)] + chunk.append(UpdateOne( + filter={'_id': expense['_id']}, + update={'$set': {f: 0 for f in fields}} + )) + if len(chunk) >= CHUNK_SIZE: + self.mongo_raw.bulk_write(chunk) + chunk = [] + if chunk: + self.mongo_raw.bulk_write(chunk) + + def downgrade(self): + pass diff --git a/ngui/ui/src/components/MlTaskDetails/TaskSummaryGrid/TaskSummaryGrid.tsx b/ngui/ui/src/components/MlTaskDetails/TaskSummaryGrid/TaskSummaryGrid.tsx index 9148900a..cde3d336 100644 --- a/ngui/ui/src/components/MlTaskDetails/TaskSummaryGrid/TaskSummaryGrid.tsx +++ b/ngui/ui/src/components/MlTaskDetails/TaskSummaryGrid/TaskSummaryGrid.tsx @@ -61,6 +61,10 @@ const TaskSummaryGrid = ({ task, recommendations, isGetRecommendationsLoading, i valueComponentProps: { value: totalSaving }, + help: { + show: true, + messageId: "projectedMonthlySavingsForRelatedInfrastructure" + }, captionMessageId: "summarySavings", relativeValueComponentType: SUMMARY_VALUE_COMPONENT_TYPES.FormattedNumber, relativeValueComponentProps: { diff --git a/ngui/ui/src/translations/en-US/app.json b/ngui/ui/src/translations/en-US/app.json index 41087802..453812b6 100644 --- a/ngui/ui/src/translations/en-US/app.json +++ b/ngui/ui/src/translations/en-US/app.json @@ -1609,6 +1609,7 @@ "prohibitedTagField": "Prohibited tag", "project": "Project", "projectedMonthlyCost": "Projected monthly cost", + "projectedMonthlySavingsForRelatedInfrastructure": "Projected monthly savings for related infrustructure", "projectedMonthlyUsage": "Projected monthly usage", "projects": "Projects", "promote": "Promote",