Enable the addition of product info for default product (#28)

* Minor fixes

* Enable the addition of product info for default product

cloud_cost_allocation/cloud_cost_allocator.py

@@ -467,16 +467,10 @@ def process_cloud_tag_selectors(self,
         cloud_tag_dict.clear()
         new_consumer_cost_items.clear()
 
-    def reset_instances(self, cost_items: list[CostItem]):
-        self.service_instances = {}
-        for cost_item in cost_items:
-            if not cost_item.is_consumer_cost_item_removed_for_cycle():
-                cost_item.set_instance_links(self)
-
     def process_default_products(self,
                                  cost_items: list[CostItem],
                                  default_product_consumer_cost_items: dict[str, list[ConsumerCostItem]],
-                                 default_product_allocation_keys: dict[str, float]):
+                                 default_product_allocation_total_keys: dict[str, float]):
 
         # Reset instances
         self.reset_instances(cost_items)
@@ -493,18 +487,17 @@ def process_default_products(self,
                     new_consumer_cost_item = self.cost_item_factory.create_consumer_cost_item()
                     new_consumer_cost_item.copy(cost_item)
                     new_consumer_cost_item.provider_cost_allocation_type += "+DefaultProduct"
+                    default_product_total_keys = default_product_allocation_total_keys[cost_item.provider_service]
                     default_product_allocation_key_ratio =\
-                        default_product_consumer_cost_item.allocation_keys[0] /\
-                        default_product_allocation_keys[cost_item.provider_service]
+                        default_product_consumer_cost_item.allocation_keys[0] / default_product_total_keys
                     for provider_meter in new_consumer_cost_item.provider_meters:
                         if "Value" in provider_meter:
                             value = provider_meter["Value"]
                             if is_float(value):
                                 provider_meter["Value"] = str(float(value) * default_product_allocation_key_ratio)
-                    new_consumer_cost_item.product = default_product_consumer_cost_item.product
-                    new_consumer_cost_item.product_dimensions =\
-                        default_product_consumer_cost_item.product_dimensions.copy()
-                    new_consumer_cost_item.allocation_keys[0] *= default_product_allocation_key_ratio
+                    self.update_default_product_from_consumer_cost_item(new_consumer_cost_item,
+                                                                        default_product_consumer_cost_item)
+                    new_consumer_cost_item.allocation_keys[0] /= default_product_total_keys
                     new_cost_items.append(new_consumer_cost_item)
             else:
                 new_cost_items.append(cost_item)
@@ -515,8 +508,8 @@ def process_default_products(self,
             for cost_item in new_cost_items:
                 if cost_item.is_self_consumption():
                     if not cost_item.get_product():
-                        cost_item.set_product(default_product)
                         cost_item.provider_cost_allocation_type += "+DefaultProduct"
+                        self.update_default_product_from_config(cost_item)
 
         # Add consumer cost items with default products for final service instances with no product
         for service_instance in self.service_instances.values():
@@ -528,25 +521,38 @@ def process_default_products(self,
                 new_consumer_cost_item.service = service_instance.service
                 new_consumer_cost_item.instance = service_instance.instance
                 new_consumer_cost_item.provider_cost_allocation_type = "DefaultProduct"
+                new_consumer_cost_item.allocation_keys[0] = 1.0
                 if service_instance.service in default_product_consumer_cost_items:
                     for default_product_consumer_cost_item\
                             in default_product_consumer_cost_items[service_instance.service]:
                         new_consumer_cost_item_with_product = self.cost_item_factory.create_consumer_cost_item()
                         new_consumer_cost_item_with_product.copy(new_consumer_cost_item)
-                        new_consumer_cost_item_with_product.product = default_product_consumer_cost_item.product
-                        new_consumer_cost_item_with_product.product_dimensions =\
-                            default_product_consumer_cost_item.product_dimensions.copy()
-                        new_consumer_cost_item_with_product.allocation_keys[0] =\
-                            default_product_consumer_cost_item.allocation_keys[0]
+                        self.update_default_product_from_consumer_cost_item(new_consumer_cost_item_with_product,
+                                                                            default_product_consumer_cost_item)
                         new_cost_items.append(new_consumer_cost_item_with_product)
                 elif default_product:
-                    new_consumer_cost_item.product = default_product
-                    new_consumer_cost_item.allocation_keys[0] = 1.0
+                    self.update_default_product_from_config(new_consumer_cost_item)
                     new_cost_items.append(new_consumer_cost_item)
 
         # Return
         return new_cost_items
 
+    def reset_instances(self, cost_items: list[CostItem]):
+        self.service_instances = {}
+        for cost_item in cost_items:
+            if not cost_item.is_consumer_cost_item_removed_for_cycle():
+                cost_item.set_instance_links(self)
+
+    def update_default_product_from_config(self, new_consumer_cost_item: ConsumerCostItem):
+        new_consumer_cost_item.product = self.cost_item_factory.config.default_product
+
+    def update_default_product_from_consumer_cost_item(self,
+                                                       new_consumer_cost_item: ConsumerCostItem,
+                                                       default_product_consumer_cost_item: ConsumerCostItem):
+        new_consumer_cost_item.product = default_product_consumer_cost_item.product
+        new_consumer_cost_item.product_dimensions = default_product_consumer_cost_item.product_dimensions.copy()
+        new_consumer_cost_item.allocation_keys[0] *= default_product_consumer_cost_item.allocation_keys[0]
+
     def visit_for_allocation(self,
                              ignore_cost_as_key: bool,
                              increment_amounts: bool,

tests/test.py

@@ -52,6 +52,14 @@ class TestConsumerCostItem(ConsumerCostItem):
     Tests the enhancements of consumer cost items
     """
 
+    __slots__ = (
+        'product_info',  # type: str
+    )
+
+    def __init__(self):
+        super().__init__()
+        self.product_info = ""
+
     def get_cost_allocation_key(self, index):
         return self.allocation_keys[index]
 
@@ -83,6 +91,19 @@ def read_item(self, line) -> TestCloudCostItem:
         return cost_item
 
 
+class TestCostAllocationKeysReader(CSV_CostAllocationKeysReader):
+
+    def __init__(self, cost_item_factory: CostItemFactory):
+        super().__init__(cost_item_factory)
+
+    def read_item(self, line) -> TestCloudCostItem:
+        cost_item = super().read_item(line)
+        # Set product info
+        if 'ProductInfo' in line:
+            cost_item.product_info = line['ProductInfo'].lower()
+        return cost_item
+
+
 class TestCsvAllocatedCostReader(CSV_AllocatedCostReader):
 
     def __init__(self, cost_item_factory: CostItemFactory):
@@ -94,6 +115,12 @@ def read_cloud_cost_item(self, line) -> TestCloudCostItem:
         cost_item.cloud = line['Cloud']
         return cost_item
 
+    def read_consumer_cost_item(self, line) -> TestConsumerCostItem:
+        cost_item = super().read_consumer_cost_item(line)
+        # Set product info
+        cost_item.product_info = line['ProductInfo']
+        return cost_item
+
 
 class TestCsvAllocatedCostWriter(CSV_AllocatedCostWriter):
 
@@ -105,21 +132,24 @@ def get_headers(self) -> list[str]:
         # - Cloud: the cloud provider name, filled for a cloud cost item
         # - IsFinalConsumption: value is Y if the cost item is a leave in the cost allocation graph, N otherwise
         headers = super().get_headers()
-        headers.extend(['Cloud', 'IsFinalConsumption'])
+        headers.extend(['Cloud', 'IsFinalConsumption', 'ProductInfo'])
         return headers
 
     def export_item_base(self, cost_item, service_instance) -> dict[str]:
         data = super().export_item_base(cost_item, service_instance)
-
         has_consumer = service_instance.is_provider()
         data['IsFinalConsumption'] = "Y" if not has_consumer or cost_item.is_self_consumption() else "N"
-
         return data
 
     def export_item_cloud(self, cost_item, service_instance) -> dict[str]:
         data = super().export_item_cloud(cost_item, service_instance)
         data['Cloud'] = cost_item.cloud
+        return data
 
+    def export_item_consumer(self, cost_item, service_instance) -> dict[str]:
+        data = super().export_item_consumer(cost_item, service_instance)
+        if cost_item.product_info:
+            data['ProductInfo'] = cost_item.product_info
         return data
 
 
@@ -139,6 +169,22 @@ def create_consumer_cost_item(self) -> ConsumerCostItem:
         return test_consumer_cost_item
 
 
+class TestCloudCostAllocator(CloudCostAllocator):
+    def __init__(self, config: Config):
+        super().__init__(config)
+
+    def update_default_product_from_consumer_cost_item(self,
+                                                       new_consumer_cost_item: ConsumerCostItem,
+                                                       default_product_consumer_cost_item: ConsumerCostItem):
+        super().update_default_product_from_consumer_cost_item(new_consumer_cost_item,
+                                                               default_product_consumer_cost_item)
+        new_consumer_cost_item.product_info = default_product_consumer_cost_item.product_info
+
+    def update_default_product_from_config(self, new_consumer_cost_item: ConsumerCostItem):
+        super().update_default_product_from_config(new_consumer_cost_item)
+        new_consumer_cost_item.product_info = "0"
+
+
 class Test(unittest.TestCase):
     """
     Runs test cases
@@ -219,12 +265,12 @@ def run_allocation(self, test: str):
 
         # Read keys
         consumer_cost_items = []
-        cost_allocation_keys_reader = CSV_CostAllocationKeysReader(cost_item_factory)
+        cost_allocation_keys_reader = TestCostAllocationKeysReader(cost_item_factory)
         allocation_keys_filename = directory + "/" + test + "/" + test + "_cost_allocation_keys.csv"
         read_csv_file(allocation_keys_filename, cost_allocation_keys_reader, consumer_cost_items)
 
         # Allocate costs
-        cloud_cost_allocator = CloudCostAllocator(cost_item_factory)
+        cloud_cost_allocator = TestCloudCostAllocator(cost_item_factory)
         cloud_cost_allocator.date_str = consumer_cost_items[0].date_str
         cloud_cost_allocator.currency = "EUR"
         assert_message = test + ": cost allocation failed"

tests/test1/test1_allocated_cost.csv

@@ -1,18 +1,18 @@
-Date,Service,Instance,Tags,AmortizedCost,OnDemandCost,Currency,ProviderService,ProviderInstance,ProviderTagSelector,ProviderCostAllocationType,ProviderCostAllocationKey,ProviderCostAllocationCloudTagSelector,Product,ProductAmortizedCost,ProductOnDemandCost,Component,Environment,ProviderMeterName1,ProviderMeterUnit1,ProviderMeterValue1,ProviderMeterName2,ProviderMeterUnit2,ProviderMeterValue2,ProductMeterName,ProductMeterUnit,ProductMeterValue,Cloud,IsFinalConsumption
-2022-03-09,container,container,"service:container,component:compute,environment:prd1,partition:0,hello\,world:hello\:\\nworld,cloud_resource_id:/az/virtualmachines/vm1,",80.0,80.0,EUR,,,,,,,,,,compute,prd1,,,,,,,,,,az,N
-2022-03-09,container,container,"service2:container,component:storage,environment:prd2,partition:1,cloud_resource_id:/az/disks/disk1,",20.0,20.0,EUR,,,,,,,,,,storage,prd2,,,,,,,,,,az,N
-2022-03-09,history,history,,5.0,5.0,EUR,container,container,"'cloud_resource_id' in globals() and __import__('re').match('/az/virtualmachines/.*',cloud_resource_id)",Key,0.25,,,,,,,cpu,core,0.25,memory,gb,0.5,,,,,N
-2022-03-09,history,history,,18.0,18.0,EUR,container,container,"'cloud_resource_id' in globals() and __import__('re').match('/az/disks/.*',cloud_resource_id)",Key,9.0,,,,,,,disk,gb,9,,,,,,,,N
-2022-03-09,history,history,,1.0,1.0,EUR,monitoring,monitoring,,Key,10.0,,,,,,,time-series,ts,10,,,,,,,,N
-2022-03-09,monitoring,monitoring,,5.0,5.0,EUR,container,container,"'cloud_resource_id' in globals() and __import__('re').match('/az/virtualmachines/.*',cloud_resource_id)",Key,0.25,,,,,grafana,,cpu,core,0.25,memory,gb,0.5,,,,,N
-2022-03-09,monitoring,monitoring,,2.0,2.0,EUR,container,container,"'cloud_resource_id' in globals() and __import__('re').match('/az/disks/.*',cloud_resource_id)",Key,1.0,,,,,prometheus,,disk,gb,1,,,,,,,,N
-2022-03-09,reporting,reporting,"service:reporting,component:batch,environment:prd,cost_center:5678,",10.0,10.0,EUR,container,container,"'cloud_resource_id' in globals() and __import__('re').match('/az/virtualmachines/.*',cloud_resource_id)",Key,0.5,,,,,batch,prd,cpu,core,0.5,memory,gb,1,,,,,N
-2022-03-09,reporting,reporting,,1.0,1.0,EUR,monitoring,monitoring,,Key,10.0,,,,,,,time-series,ts,10,,,,,,,,N
-2022-03-09,reporting,reporting,,12.0,12.0,EUR,history,history,,Key,1.0,,,,,,,records,rec,500,,,,,,,,N
-2022-03-09,reporting,reporting,,20.700000000000003,20.700000000000003,EUR,reporting,reporting,,Key,9.0,,b2b,20.700000000000003,20.700000000000003,,,reports,rep,9,,,,monthly active users,usr,100,,Y
-2022-03-09,reporting,reporting,,2.3000000000000003,2.3000000000000003,EUR,reporting,reporting,,Key,1.0,,b2c,2.3000000000000003,2.3000000000000003,,,reports,rep,1,,,,monthly active users,usr,900,,Y
-2022-03-09,web,web,"cost_center:1234,",60.0,60.0,EUR,container,container,"'cloud_resource_id' in globals() and __import__('re').match('/az/virtualmachines/.*',cloud_resource_id)",Key,3.0,,,,,apache,prd,cpu,core,3,memory,gb,6,,,,,N
-2022-03-09,web,web,,5.0,5.0,EUR,monitoring,monitoring,,Key,50.0,,,,,,,time-series,ts,50,,,,,,,,N
-2022-03-09,web,web,,12.0,12.0,EUR,history,history,,Key,1.0,,,,,,,records,rec,500,,,,,,,,N
-2022-03-09,web,web,,7.699999999999999,7.699999999999999,EUR,web,web,,Key,100.0,,b2b,7.699999999999999,7.699999999999999,,,http requests,req,100,,,,monthly active users,,,,Y
-2022-03-09,web,web,,69.3,69.3,EUR,web,web,,Key,900.0,,b2c,69.3,69.3,,,http requests,req,900,,,,monthly active users,,,,Y
+Date,Service,Instance,Tags,AmortizedCost,OnDemandCost,Currency,ProviderService,ProviderInstance,ProviderTagSelector,ProviderCostAllocationType,ProviderCostAllocationKey,ProviderCostAllocationCloudTagSelector,Product,ProductAmortizedCost,ProductOnDemandCost,Component,Environment,ProviderMeterName1,ProviderMeterUnit1,ProviderMeterValue1,ProviderMeterName2,ProviderMeterUnit2,ProviderMeterValue2,ProductMeterName,ProductMeterUnit,ProductMeterValue,Cloud,IsFinalConsumption,ProductInfo
+2022-03-09,container,container,"service:container,component:compute,environment:prd1,partition:0,hello\,world:hello\:\\nworld,cloud_resource_id:/az/virtualmachines/vm1,",80.0,80.0,EUR,,,,,,,,,,compute,prd1,,,,,,,,,,az,N,
+2022-03-09,container,container,"service2:container,component:storage,environment:prd2,partition:1,cloud_resource_id:/az/disks/disk1,",20.0,20.0,EUR,,,,,,,,,,storage,prd2,,,,,,,,,,az,N,
+2022-03-09,history,history,,5.0,5.0,EUR,container,container,"'cloud_resource_id' in globals() and __import__('re').match('/az/virtualmachines/.*',cloud_resource_id)",Key,0.25,,,,,,,cpu,core,0.25,memory,gb,0.5,,,,,N,
+2022-03-09,history,history,,18.0,18.0,EUR,container,container,"'cloud_resource_id' in globals() and __import__('re').match('/az/disks/.*',cloud_resource_id)",Key,9.0,,,,,,,disk,gb,9,,,,,,,,N,
+2022-03-09,history,history,,1.0,1.0,EUR,monitoring,monitoring,,Key,10.0,,,,,,,time-series,ts,10,,,,,,,,N,
+2022-03-09,monitoring,monitoring,,5.0,5.0,EUR,container,container,"'cloud_resource_id' in globals() and __import__('re').match('/az/virtualmachines/.*',cloud_resource_id)",Key,0.25,,,,,grafana,,cpu,core,0.25,memory,gb,0.5,,,,,N,
+2022-03-09,monitoring,monitoring,,2.0,2.0,EUR,container,container,"'cloud_resource_id' in globals() and __import__('re').match('/az/disks/.*',cloud_resource_id)",Key,1.0,,,,,prometheus,,disk,gb,1,,,,,,,,N,
+2022-03-09,reporting,reporting,"service:reporting,component:batch,environment:prd,cost_center:5678,",10.0,10.0,EUR,container,container,"'cloud_resource_id' in globals() and __import__('re').match('/az/virtualmachines/.*',cloud_resource_id)",Key,0.5,,,,,batch,prd,cpu,core,0.5,memory,gb,1,,,,,N,
+2022-03-09,reporting,reporting,,1.0,1.0,EUR,monitoring,monitoring,,Key,10.0,,,,,,,time-series,ts,10,,,,,,,,N,
+2022-03-09,reporting,reporting,,12.0,12.0,EUR,history,history,,Key,1.0,,,,,,,records,rec,500,,,,,,,,N,
+2022-03-09,reporting,reporting,,20.700000000000003,20.700000000000003,EUR,reporting,reporting,,Key,9.0,,b2b,20.700000000000003,20.700000000000003,,,reports,rep,9,,,,monthly active users,usr,100,,Y,
+2022-03-09,reporting,reporting,,2.3000000000000003,2.3000000000000003,EUR,reporting,reporting,,Key,1.0,,b2c,2.3000000000000003,2.3000000000000003,,,reports,rep,1,,,,monthly active users,usr,900,,Y,
+2022-03-09,web,web,"cost_center:1234,",60.0,60.0,EUR,container,container,"'cloud_resource_id' in globals() and __import__('re').match('/az/virtualmachines/.*',cloud_resource_id)",Key,3.0,,,,,apache,prd,cpu,core,3,memory,gb,6,,,,,N,
+2022-03-09,web,web,,5.0,5.0,EUR,monitoring,monitoring,,Key,50.0,,,,,,,time-series,ts,50,,,,,,,,N,
+2022-03-09,web,web,,12.0,12.0,EUR,history,history,,Key,1.0,,,,,,,records,rec,500,,,,,,,,N,
+2022-03-09,web,web,,7.699999999999999,7.699999999999999,EUR,web,web,,Key,100.0,,b2b,7.699999999999999,7.699999999999999,,,http requests,req,100,,,,monthly active users,,,,Y,
+2022-03-09,web,web,,69.3,69.3,EUR,web,web,,Key,900.0,,b2c,69.3,69.3,,,http requests,req,900,,,,monthly active users,,,,Y,