Improved logging and clear memory from large temporary dictionaries (#12

) * Improved logging and clear memory from large temporary dictionaries * Version 0.2.5
AmadeusITGroup · Dec 12, 2022 · 4576d2a · 4576d2a
1 parent 2e97500
commit 4576d2a
Show file tree

Hide file tree

Showing 3 changed files with 23 additions and 14 deletions.
diff --git a/cloud_cost_allocation/cloud_cost_allocator.py b/cloud_cost_allocation/cloud_cost_allocator.py
@@ -51,14 +51,17 @@ def allocate(self, consumer_cost_items: list[ConsumerCostItem], cloud_cost_items
         cost_items.extend(new_consumer_cost_items)
 
         # Check all dates are the same
-        date_str = ""
+        reference_date_str = ""
+        different_date_str_list = []
         for cost_item in cost_items:
             if cost_item.date_str:
-                date_str = cost_item.date_str
-                break # We just need to find one as a reference
+                reference_date_str = cost_item.date_str
+                break  # We just need to find one date as a reference
         for cost_item in cost_items:
-            if date_str != cost_item.date_str:
-                error("Found cost items with different dates: " + date_str + " and " + cost_item.date_str)
+            if cost_item.date_str != reference_date_str and cost_item.date_str not in different_date_str_list:
+                different_date_str_list.append(cost_item.date_str)
+        for different_date_str in different_date_str_list:
+            error("Found cost items with different dates: " + reference_date_str + " and " + different_date_str)
 
         # Check all currencies are the same and populate missing ones (currencies are missing for consumer
         # cost items)
@@ -77,6 +80,7 @@ def allocate(self, consumer_cost_items: list[ConsumerCostItem], cloud_cost_items
 
         # Break cycles
         # When a cycle is broken, ConsumerCostItem.is_removed_from_cycle is set to True at the cycle break point
+        info("Breaking cycles, for date " + reference_date_str)
         if not self.break_cycles(cost_items):
             return False
 
@@ -91,26 +95,26 @@ def allocate(self, consumer_cost_items: list[ConsumerCostItem], cloud_cost_items
 
             # Compute service amortized cost, ignoring the keys that are using cost, and then
             # set these keys from the computed cost
-            info("Allocating costs, ignoring keys that are costs")
+            info("Allocating costs, ignoring keys that are costs, for date " + reference_date_str)
             self.visit_for_allocation(True, True, False)
             for service_instance in self.service_instances.values():
                 for cost_item in service_instance.cost_items:
                     cost_item.set_cost_as_key(service_instance.cost)
 
             # Allocate amortized costs for services
-            info("Allocating amortized costs for services")
+            info("Allocating amortized costs for services, for date " + reference_date_str)
             self.visit_for_allocation_and_set_allocated_cost(True, False)
 
             # Allocate on-demand costs for services
-            info("Allocating on-demand costs for services")
+            info("Allocating on-demand costs for services, for date " + reference_date_str)
             self.visit_for_allocation_and_set_allocated_cost(False, False)
 
             # Allocate amortized costs for products
-            info("Allocating amortized costs for products")
+            info("Allocating amortized costs for products, for date " + reference_date_str)
             self.visit_for_allocation_and_set_allocated_cost(True, True)
 
             # Allocate on-demand costs for products
-            info("Allocating on-demand costs for products")
+            info("Allocating on-demand costs for products, for date " + reference_date_str)
             self.visit_for_allocation_and_set_allocated_cost(False, True)
 
         except CycleException:
@@ -412,6 +416,11 @@ def process_cloud_tag_selectors(self,
                   "' of ProviderService '" + provider_service +
                   "' and of ProviderInstance '" + provider_instance + "'")
 
+        # Clean-up for the sake of memory
+        evaluation_error_dict.clear()
+        cloud_tag_dict.clear()
+        new_consumer_cost_items.clear()
+
     def visit_for_allocation_and_set_allocated_cost(self, is_amortized_cost: bool, is_for_products: bool) -> None:
 
         # Visit, using cost keys, which have been previously computed

diff --git a/cloud_cost_allocation/cost_items.py b/cloud_cost_allocation/cost_items.py
@@ -2,7 +2,7 @@
 
 from abc import ABC, abstractmethod
 from io import StringIO
-from logging import info, error
+from logging import debug, info, error
 import re
 import sys
 
@@ -547,9 +547,9 @@ def compute_provider_tag_selector_costs(self) -> None:
             if not item.is_self_consumption() and item.nb_matching_provider_tag_selectors != 1:
                 is_partition = False
 
-        # For info
+        # For debug
         if not is_partition:
-            info("Provider tag selectors of provider service instance " +
+            debug("Provider tag selectors of provider service instance " +
                  self.service + "." + self.instance +
                  " do not form a partition")
 

diff --git a/setup.py b/setup.py
@@ -10,7 +10,7 @@
 # Setup
 setup(
     name='cloud-cost-allocation',
-    version='0.2.4',
+    version='0.2.5',
     description='Python library for shared, hierarchical cost allocation based on user-defined metrics.',
     long_description=readme,
     long_description_content_type='text/markdown',