From 2e97500fd8adeac5b8698d4a9e8559ff8cb15625 Mon Sep 17 00:00:00 2001 From: Marc PERREAUT Date: Fri, 25 Nov 2022 18:27:37 +0100 Subject: [PATCH] Fixed consumer service instance unification issue in cloud tag selector processing (#11) * Fixed consuner service instance unification issue in cloud tag selector processing * Version 0.2.4 --- cloud_cost_allocation/cloud_cost_allocator.py | 44 ++++++++++--------- setup.py | 2 +- tests/test3/test3_allocated_cost.csv | 5 ++- tests/test3/test3_cloud_cost.csv | 5 ++- 4 files changed, 31 insertions(+), 25 deletions(-) diff --git a/cloud_cost_allocation/cloud_cost_allocator.py b/cloud_cost_allocation/cloud_cost_allocator.py index 34f43f3..894e88d 100644 --- a/cloud_cost_allocation/cloud_cost_allocator.py +++ b/cloud_cost_allocation/cloud_cost_allocator.py @@ -310,6 +310,12 @@ def process_cloud_tag_selectors(self, cloud_tag_dict[tag_string] = cloud_cost_item_list cloud_cost_item_list.append(cloud_cost_item) + # Create a unique consumer cost item for a given consumer service instance and for a given cloud tag + # selector consumer cost item + # TODO: Create different consumer cost items for different consumer cost item dimensions + new_consumer_cost_items = {} + # Key is (consumer service instance id, number of cloud tag selector consumer in input list) + # Process cloud tag dict for cloud_cost_item_list in cloud_tag_dict.values(): @@ -319,27 +325,26 @@ def process_cloud_tag_selectors(self, eval_globals_dict[re.sub(r'[^a-z0-9_]', '_', key)] = value # Process consumer cost items + selector_nb = 0 for cloud_tag_selector_consumer_cost_item in cloud_tag_selector_consumer_cost_items: - - # New consumer cost items created for this tag context and for this cloud tag selector - # There is a unique consumer cost item for a given consumer service instance - # TODO: Create different consumer cost items for different dimensions - new_consumer_cost_items = {} # Key is (consumer) service instance id + selector_nb += 1 # Check if cloud selector match + provider_service = cloud_tag_selector_consumer_cost_item.provider_service + provider_instance = cloud_tag_selector_consumer_cost_item.provider_instance + cloud_tag_selector = cloud_tag_selector_consumer_cost_item.provider_cost_allocation_cloud_tag_selector match = False try: # Eval is dangerous. Considerations: # - Possibly forbid cloud tag selectors # - Possibly whitelist cloud tag selectors using pattern matching - match = eval(cloud_tag_selector_consumer_cost_item.provider_cost_allocation_cloud_tag_selector, - eval_globals_dict, {}) + match = eval(cloud_tag_selector, eval_globals_dict, {}) except: exception = sys.exc_info()[0] error_key =\ - cloud_tag_selector_consumer_cost_item.provider_service + chr(10) +\ - cloud_tag_selector_consumer_cost_item.provider_instance + chr(10) +\ - cloud_tag_selector_consumer_cost_item.provider_cost_allocation_cloud_tag_selector + chr(10) +\ + provider_service + chr(10) +\ + provider_instance + chr(10) +\ + cloud_tag_selector + chr(10) +\ str(exception) if error_key in evaluation_error_dict: error_count = evaluation_error_dict[error_key] @@ -355,12 +360,13 @@ def process_cloud_tag_selectors(self, if cloud_cost_item.service != cloud_tag_selector_consumer_cost_item.provider_service: # Check if matching service instance was already processed - consumer_cost_item_id =\ + consumer_cost_item_key =\ + str(selector_nb) + chr(10) +\ ServiceInstance.get_id(cloud_cost_item.service, cloud_cost_item.instance) - if consumer_cost_item_id in new_consumer_cost_items: + if consumer_cost_item_key in new_consumer_cost_items: # Increase existing cost allocation key with the amortized cost of this cloud cost item - new_consumer_cost_item = new_consumer_cost_items[consumer_cost_item_id] + new_consumer_cost_item = new_consumer_cost_items[consumer_cost_item_key] new_consumer_cost_item.provider_cost_allocation_key +=\ cloud_cost_item.cloud_amortized_cost @@ -376,10 +382,8 @@ def process_cloud_tag_selectors(self, new_consumer_cost_item.service = cloud_cost_item.service new_consumer_cost_item.instance = cloud_cost_item.instance new_consumer_cost_item.tags = cloud_tag_selector_consumer_cost_item.tags.copy() - new_consumer_cost_item.provider_service =\ - cloud_tag_selector_consumer_cost_item.provider_service - new_consumer_cost_item.provider_instance =\ - cloud_tag_selector_consumer_cost_item.provider_instance + new_consumer_cost_item.provider_service = provider_service + new_consumer_cost_item.provider_instance = provider_instance # TODO: split and dispatch the provider meter values new_consumer_cost_item.provider_meters =\ cloud_tag_selector_consumer_cost_item.provider_meters.copy() @@ -389,11 +393,11 @@ def process_cloud_tag_selectors(self, cloud_tag_selector_consumer_cost_item.provider_tag_selector new_consumer_cost_item.provider_cost_allocation_key =\ cloud_cost_item.cloud_amortized_cost - new_consumer_cost_item.provider_cost_allocation_cloud_tag_selector = \ - cloud_tag_selector_consumer_cost_item.provider_cost_allocation_cloud_tag_selector + new_consumer_cost_item.provider_cost_allocation_cloud_tag_selector =\ + cloud_tag_selector # Add new consumer cost item - new_consumer_cost_items[consumer_cost_item_id] = new_consumer_cost_item + new_consumer_cost_items[consumer_cost_item_key] = new_consumer_cost_item cost_items.append(new_consumer_cost_item) # Log evaluation errors diff --git a/setup.py b/setup.py index e77f489..4b3e8db 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ # Setup setup( name='cloud-cost-allocation', - version='0.2.3', + version='0.2.4', description='Python library for shared, hierarchical cost allocation based on user-defined metrics.', long_description=readme, long_description_content_type='text/markdown', diff --git a/tests/test3/test3_allocated_cost.csv b/tests/test3/test3_allocated_cost.csv index 9d43e39..f429909 100644 --- a/tests/test3/test3_allocated_cost.csv +++ b/tests/test3/test3_allocated_cost.csv @@ -7,10 +7,11 @@ Date,Service,Instance,Tags,AmortizedCost,OnDemandCost,Currency,ProviderService,P 2022-03-11,finops,reservation,,10.0,0.0,EUR,,,,,,,,,,unused,az,N 2022-03-11,shared1,shared1,"service:shared1,reservation_zone:green,cloud_resource_id:resourceid1,",100.0,150.0,EUR,,,,,,,,,,,az,N 2022-03-11,shared1,shared1,,10.0,0.0,EUR,finops,reservation,,CloudTagSelector,100.0,'reservation_zone' in globals() and reservation_zone == 'green',,,,,,N -2022-03-11,shared2,instance1,"service:shared2,instance:instance1,reservation_zone:green,cloud_resource_id:resourceid2,",20.0,30.0,EUR,,,,,,,,,,,az,N +2022-03-11,shared2,instance1,"service:shared2,instance:instance1,reservation_zone:green,cloud_resource_id:resourceid2,",10.0,15.0,EUR,,,,,,,,,,,az,N +2022-03-11,shared2,instance1,"service:shared2,instance:instance1,reservation_zone:green,cloud_resource_id:resourceid3,",10.0,15.0,EUR,,,,,,,,,,,az,N 2022-03-11,shared2,instance1,"consumer_service:application1,consumer_instance:instance1,consumer_component:component1,",11.0,15.0,EUR,shared1,shared1,,Cost,22.0,,,,,,,N 2022-03-11,shared2,instance1,,2.0,0.0,EUR,finops,reservation,,CloudTagSelector,20.0,'reservation_zone' in globals() and reservation_zone == 'green',,,,,,N -2022-03-11,shared2,instance2,"service:shared2,instance:instance2,reservation_zone:green,cloud_resource_id:resourceid3,",80.0,120.0,EUR,,,,,,,,,,,az,N +2022-03-11,shared2,instance2,"service:shared2,instance:instance2,reservation_zone:green,cloud_resource_id:resourceid4,",80.0,120.0,EUR,,,,,,,,,,,az,N 2022-03-11,shared2,instance2,"consumer_service:application2,consumer_instance:instance2,",44.0,60.0,EUR,shared1,shared1,,Cost,88.0,,,,,,,N 2022-03-11,shared2,instance2,,8.0,0.0,EUR,finops,reservation,,CloudTagSelector,80.0,'reservation_zone' in globals() and reservation_zone == 'green',,,,,,N 2022-03-11,x,x,"cloud_resource_id:resourceid0,",100.0,150.0,EUR,,,,,,,,,,,az,Y diff --git a/tests/test3/test3_cloud_cost.csv b/tests/test3/test3_cloud_cost.csv index a012e3b..612445f 100644 --- a/tests/test3/test3_cloud_cost.csv +++ b/tests/test3/test3_cloud_cost.csv @@ -1,8 +1,9 @@ Date,Tags,CostInBillingCurrency,BillingCurrencyCode,ChargeType,ResourceId,ReservationId,Quantity,UnitPrice 03/11/2022,,100,EUR,Usage,ResourceId0,ReservationId0,2,75 03/11/2022,"""service"": ""shared1"""",""""reservation_zone"": ""green""",100,EUR,Usage,ResourceId1,ReservationId1,5,30 -03/11/2022,"""service"": ""shared2"""",""""instance"": ""instance1"""",""""reservation_zone"": ""green""",20,EUR,Usage,ResourceId2,ReservationId2,3,10 -03/11/2022,"""service"": ""shared2"""",""""instance"": ""instance2"""",""""reservation_zone"": ""green""",80,EUR,Usage,ResourceId3,ReservationId3,3,40 +03/11/2022,"""service"": ""shared2"""",""""instance"": ""instance1"""",""""reservation_zone"": ""green""",10,EUR,Usage,ResourceId2,ReservationId2,1.5,10 +03/11/2022,"""service"": ""shared2"""",""""instance"": ""instance1"""",""""reservation_zone"": ""green""",10,EUR,Usage,ResourceId3,ReservationId2,1.5,10 +03/11/2022,"""service"": ""shared2"""",""""instance"": ""instance2"""",""""reservation_zone"": ""green""",80,EUR,Usage,ResourceId4,ReservationId3,3,40 03/11/2022,,10,EUR,UnusedReservation,,ReservationId0,, 03/11/2022,,5,EUR,UnusedReservation,,ReservationId1,, 03/11/2022,,5,EUR,UnusedReservation,,ReservationId2,,