Version 1.0.8: fixed tag serialization and deserialization for specia…

…l characters (#21)
AmadeusITGroup · Nov 2, 2023 · c783951 · c783951
1 parent c9bd8a2
commit c783951
Show file tree

Hide file tree

Showing 9 changed files with 58 additions and 44 deletions.
diff --git a/cloud_cost_allocation/reader/csv_allocated_cost_reader.py b/cloud_cost_allocation/reader/csv_allocated_cost_reader.py
@@ -5,6 +5,7 @@
 
 from cloud_cost_allocation.cost_items import CloudCostItem, ConsumerCostItem, CostItem, CostItemFactory
 from cloud_cost_allocation.reader.base_reader import GenericReader
+from cloud_cost_allocation.utils import utils
 
 
 class CSV_AllocatedCostReader(GenericReader):
@@ -32,16 +33,7 @@ def initialize_cost_item(self, cost_item: CostItem, line):
             cost_item.dimensions[dimension] = line[dimension]
         tag_str = line["Tags"]
         if tag_str:
-            tags = tag_str.split(',')
-            for tag in tags:
-                if tag:
-                    key_value_match = re.match("([^:]+):([^:]*)", tag)
-                    if key_value_match:
-                        key = key_value_match.group(1).strip().lower()
-                        value = key_value_match.group(2).strip().lower()
-                        cost_item.tags[key] = value
-                    else:
-                        error("Unexpected tag format: '" + tag + "'")
+            cost_item.tags = utils.deserialize_tags(tag_str)
 
         # Amounts
         index = 0

diff --git a/cloud_cost_allocation/reader/csv_cost_allocation_keys_reader.py b/cloud_cost_allocation/reader/csv_cost_allocation_keys_reader.py
@@ -107,16 +107,7 @@ def read_item(self, line) -> ConsumerCostItem:
         if 'ConsumerTags' in line:
             consumer_tags = line["ConsumerTags"]
             if consumer_tags:
-                tags = consumer_tags.split(',')
-                for tag in tags:
-                    if tag:
-                        key_value_match = re.match("([^:]+):([^:]*)", tag)
-                        if key_value_match:
-                            key = key_value_match.group(1).strip().lower()
-                            value = key_value_match.group(2).strip().lower()
-                            consumer_cost_item.tags[key] = value
-                        else:
-                            error("Unexpected consumer tag format: '" + tag + "'")
+                consumer_cost_item.tags = utils.deserialize_tags(consumer_tags)
 
         # Populate consumer cost item info from consumer tags
         # Note: if both consumer tags and other consumer columns (ConsumerService, ConsumerInstance,

diff --git a/cloud_cost_allocation/utils/utils.py b/cloud_cost_allocation/utils/utils.py
@@ -3,9 +3,30 @@
 
 @author: marc.diensberg
 '''
-import validators
+
+import csv
+import io
+import logging
+import re
 import requests
-from csv import DictReader, DictWriter
+import validators
+
+
+def deserialize_tags(tags_str: str):
+    deserialized = re.sub(r'([^\\]):', r'\g<1>\a', tags_str)  # Replace non-escaped : by \a (control character)
+    deserialized = re.sub(r'([^\\]),', r'\g<1>\b', deserialized)  # Replace non-escaped , by \b (control character)
+    deserialized = re.sub(r'\\([:,\\])', r'\g<1>', deserialized)  # Remove escape characters
+    tags = {}
+    for tag_str in deserialized.split('\b'):
+        if tag_str:
+            key_value_match = re.match("([^\a]+)\a([^\a]*)", tag_str)
+            if key_value_match:
+                key = key_value_match.group(1).strip().lower()
+                value = key_value_match.group(2).strip().lower()
+                tags[key] = value
+            else:
+                logging.error("Unexpected tag format: '" + tag_str + "'")
+    return tags
 
 
 def is_float(value):
@@ -15,6 +36,7 @@ def is_float(value):
     except(TypeError, ValueError):
         return False
 
+
 def read_csv(uri, reader, cost_items):
     if validators.url(uri):
         read_csv_url(uri, reader, cost_items)
@@ -24,23 +46,34 @@ def read_csv(uri, reader, cost_items):
 
 def read_csv_file(uri, reader, cost_items):
     with open(uri, 'r') as text_io:
-        dictReader = DictReader(text_io)
-        reader.read(cost_items, dictReader)
+        dict_reader = csv.DictReader(text_io)
+        reader.read(cost_items, dict_reader)
 
 
 def read_csv_url(uri, reader, cost_items):
     response = requests.get(uri)
-    dictReader = DictReader(response.iter_lines())
-    reader.read(cost_items, dictReader)
+    dict_reader = csv.DictReader(response.iter_lines())
+    reader.read(cost_items, dict_reader)
+
+
+def serialize_tag_key_or_value(tag_key_or_value: str):
+    return re.sub(r'([,:\\])', r'\\\g<1>', tag_key_or_value)
+
+
+def serialize_tags(tags: dict[str, str]):
+    tags_str = io.StringIO()
+    for key, value in tags.items():
+        tags_str.write(serialize_tag_key_or_value(key) + ":" + serialize_tag_key_or_value(value) + ",")
+    return tags_str.getvalue()
 
 
 def write_csv_file(uri, writer):
     # Write allocated costs
-    with open(uri, 'w', newline='') as outstream:
+    with open(uri, 'w', newline='') as out_stream:
         # Open CSV file and write header
-        dictWriter = DictWriter(outstream,
-                                fieldnames=writer.get_headers(),
-                                restval='',
-                                extrasaction='ignore')
-        dictWriter.writeheader()
-        writer.write(dictWriter)
+        dict_writer = csv.DictWriter(out_stream,
+                                     fieldnames=writer.get_headers(),
+                                     restval='',
+                                     extrasaction='ignore')
+        dict_writer.writeheader()
+        writer.write(dict_writer)
diff --git a/cloud_cost_allocation/writer/csv_allocated_cost_writer.py b/cloud_cost_allocation/writer/csv_allocated_cost_writer.py
@@ -3,11 +3,12 @@
 
 @author: marc.diensberg
 '''
-from io import StringIO
 
 from cloud_cost_allocation.cost_items import Config, ServiceInstance
 from cloud_cost_allocation.writer.base_writer import GenericWriter
 
+from cloud_cost_allocation.utils import utils
+
 
 class CSV_AllocatedCostWriter(GenericWriter):
     '''
@@ -90,10 +91,7 @@ def export_item_base(self, cost_item, service_instance) -> dict[str]:
         data['Date'] = cost_item.date_str
         data['Service'] = cost_item.service
         data['Instance'] = cost_item.instance
-        tags_str = StringIO()
-        for key, value in cost_item.tags.items():
-            tags_str.write(key + ":" + value + ",")
-        data['Tags'] = tags_str.getvalue()
+        data['Tags'] = utils.serialize_tags(cost_item.tags)
         data['AmortizedCost'] = str(cost_item.amounts[0])
         data['OnDemandCost'] = str(cost_item.amounts[1])
         data['Currency'] = cost_item.currency

diff --git a/setup.py b/setup.py
@@ -10,7 +10,7 @@
 # Setup
 setup(
     name='cloud-cost-allocation',
-    version='1.0.7',
+    version='1.0.8',
     description='Python library for shared, hierarchical cost allocation based on user-defined usage metrics.',
     long_description=readme,
     long_description_content_type='text/markdown',

diff --git a/tests/test1/test1_allocated_cost.csv b/tests/test1/test1_allocated_cost.csv
@@ -1,5 +1,5 @@
 Date,Service,Instance,Tags,AmortizedCost,OnDemandCost,Currency,ProviderService,ProviderInstance,ProviderTagSelector,ProviderCostAllocationType,ProviderCostAllocationKey,ProviderCostAllocationCloudTagSelector,Product,ProductAmortizedCost,ProductOnDemandCost,Component,Environment,ProviderMeterName1,ProviderMeterUnit1,ProviderMeterValue1,ProviderMeterName2,ProviderMeterUnit2,ProviderMeterValue2,ProductMeterName,ProductMeterUnit,ProductMeterValue,Cloud,IsFinalConsumption
-2022-03-09,container,container,"service:container,component:compute,environment:prd1,partition:0,cloud_resource_id:/az/virtualmachines/vm1,",80.0,80.0,EUR,,,,,,,,,,compute,prd1,,,,,,,,,,az,N
+2022-03-09,container,container,"service:container,component:compute,environment:prd1,partition:0,hello\,world:hello\:\\nworld,cloud_resource_id:/az/virtualmachines/vm1,",80.0,80.0,EUR,,,,,,,,,,compute,prd1,,,,,,,,,,az,N
 2022-03-09,container,container,"service2:container,component:storage,environment:prd2,partition:1,cloud_resource_id:/az/disks/disk1,",20.0,20.0,EUR,,,,,,,,,,storage,prd2,,,,,,,,,,az,N
 2022-03-09,history,history,,5.0,5.0,EUR,container,container,"'cloud_resource_id' in globals() and __import__('re').match('/az/virtualmachines/.*',cloud_resource_id)",Key,0.25,,,,,,,cpu,core,0.25,memory,gb,0.5,,,,,N
 2022-03-09,history,history,,18.0,18.0,EUR,container,container,"'cloud_resource_id' in globals() and __import__('re').match('/az/disks/.*',cloud_resource_id)",Key,9.0,,,,,,,disk,gb,9,,,,,,,,N

diff --git a/tests/test1/test1_cloud_cost.csv b/tests/test1/test1_cloud_cost.csv
@@ -1,3 +1,3 @@
 Date,LineNumber,Tags,CostInBillingCurrency,BillingCurrencyCode,PricingModel,ChargeType,ResourceId
-03/09/2022,1,"""service"": ""container"""",""""component"": ""compute"""",""""environment"": ""prd1"""",""""partition"": ""0""",80,EUR,OnDemand,Usage,/Az/virtualMachines/vm1
-03/09/2022,1,"""service2"": ""container"""",""""component"": ""storage"""",""""environment"": ""prd2"""",""""partition"": ""1""",20,EUR,OnDemand,Usage,/Az/disks/disk1
+03/09/2022,1,"""service"": ""container"",""component"": ""compute"",""environment"": ""prd1"",""partition"": ""0"",""hello,world"": ""hello:\nworld""",80,EUR,OnDemand,Usage,/Az/virtualMachines/vm1
+03/09/2022,1,"""service2"": ""container"",""component"": ""storage"",""environment"": ""prd2"",""partition"": ""1""",20,EUR,OnDemand,Usage,/Az/disks/disk1
diff --git a/tests/test7/test7_allocated_cost.csv b/tests/test7/test7_allocated_cost.csv
@@ -1,5 +1,5 @@
 Date,Service,Instance,Tags,AmortizedCost,OnDemandCost,Currency,ProviderService,ProviderInstance,ProviderTagSelector,ProviderCostAllocationCloudTagSelector,ProviderCostAllocationType,ProviderCostAllocationKey,Product,ProductAmortizedCost,ProductOnDemandCost,Cloud,IsFinalConsumption
-26/09/2023,application1,application1,,80,80,EUR,container,container,,,,80,,,,,N
+26/09/2023,application1,application1,"""hello\,world"": ""hello\:\\nworld""",80,80,EUR,container,container,,,,80,,,,,N
 26/09/2023,application1,application1,,64,64,EUR,application1,application1,,,,80,product1,64,64,,Y
 26/09/2023,application1,application1,,16,16,EUR,application1,application1,,,,20,product2,16,16,,Y
 26/09/2023,application2,application2,,20,20,EUR,container,container,,,,20,,,,,N

diff --git a/tests/test7/test7_further_allocated_cost.csv b/tests/test7/test7_further_allocated_cost.csv
@@ -1,5 +1,5 @@
 Date,Service,Instance,Tags,AmortizedCost,OnDemandCost,Currency,ProviderService,ProviderInstance,ProviderTagSelector,ProviderCostAllocationType,ProviderCostAllocationKey,ProviderCostAllocationCloudTagSelector,Product,ProductAmortizedCost,ProductOnDemandCost,Co2,ElectricityPower,ProductCo2,ProductElectricityPower,Co2Key,Cloud,IsFinalConsumption
-26/09/2023,application1,application1,,80.0,80.0,EUR,container,container,,,80.0,,,,,6.0,0.6,,,60,,N
+26/09/2023,application1,application1,"""hello\,world"":""hello\:\\nworld"",",80.0,80.0,EUR,container,container,,,80.0,,,,,6.0,0.6,,,60,,N
 26/09/2023,application1,application1,,64.0,64.0,EUR,application1,application1,,,80.0,,product1,64.0,64.0,4.800000000000001,0.48,4.800000000000001,0.48,80.0,,Y
 26/09/2023,application1,application1,,16.0,16.0,EUR,application1,application1,,,20.0,,product2,16.0,16.0,1.2000000000000002,0.12,1.2000000000000002,0.12,20.0,,Y
 26/09/2023,application2,application2,,20.0,20.0,EUR,container,container,,,20.0,,,,,4.0,0.4,,,40,,N