From 8f67b61462c41e9c14ae7b194f818547ed2e7b34 Mon Sep 17 00:00:00 2001 From: emlunde Date: Fri, 29 Nov 2024 13:17:00 +0100 Subject: [PATCH 1/3] Fix: Corrected inlcusive max date to also work when with uncapped minimum date in range term --- web/src/SAP/src/services/search/transpiler.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/web/src/SAP/src/services/search/transpiler.py b/web/src/SAP/src/services/search/transpiler.py index 0c8b85d..bb14633 100644 --- a/web/src/SAP/src/services/search/transpiler.py +++ b/web/src/SAP/src/services/search/transpiler.py @@ -105,17 +105,17 @@ def convert_type(value): def structure_ranged(field, node): min_op = "$gte" if node.inclusive == "left" or node.inclusive == "both" else "$gt" max_op = "$lte" if node.inclusive == "right" or node.inclusive == "both" else "$lt" - if node.term_min == "*": - return {max_op: convert_type(node.term_max)} - if node.term_max == "*": - return {min_op: convert_type(node.term_min)} - #if the hours, minutes and seconds are not in the search like this 2022-04-08T09:01:07 it is assumed that the entire day is intended to be included - #default with the specific time of day not specified it is as if they are 00, which would exclude all records from during that day, which is not the behavior we expect is wanted - max_term = convert_type(node.term_max) + #default with the specific time of day not specified it is as if they are 00, which would exclude all records from during that day, which is not the behavior we expect is wanted + max_term = convert_type(node.term_max) if type(max_term) == datetime and max_op =="$lte" and max_term.hour == 0 and max_term.minute == 0 and max_term.second == 0: max_term = max_term + timedelta(days = 1 ) - timedelta(seconds = 1) + if node.term_min == "*": + return {max_op: max_term} + if node.term_max == "*": + return {min_op: convert_type(node.term_min)} + return {min_op: convert_type(node.term_min), max_op: max_term} From ebb104e02fb4aa05915fce50fe893eaa9233e282 Mon Sep 17 00:00:00 2001 From: emlunde Date: Fri, 29 Nov 2024 15:13:53 +0100 Subject: [PATCH 2/3] Fix: Exact search for pii columns now works for other types than string as well --- web/src/SAP/src/services/search/transpiler.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/web/src/SAP/src/services/search/transpiler.py b/web/src/SAP/src/services/search/transpiler.py index bb14633..9945326 100644 --- a/web/src/SAP/src/services/search/transpiler.py +++ b/web/src/SAP/src/services/search/transpiler.py @@ -69,6 +69,10 @@ def coerce_term(term: str): def structure_wildcard(field, node): + if field in pii_columns(): + if field == "cpr_nr": # because cpr is saved as a string in the DB, so don't convert even when consists of only digits + return node.term + return coerce_term(node.term) coerced = coerce_term(node.term) if isinstance(coerced, str): return check_for_wildcard(field, coerced) @@ -93,7 +97,9 @@ def is_float_string(value): return False def convert_type(value): - if is_float_string(value): + if value.isdigit(): + return int(value) + elif is_float_string(value): return float(value) elif value.isdigit(): return int(value) From 673d2d44b0b88260bb100d9ab5d43938cea53b48 Mon Sep 17 00:00:00 2001 From: emlunde Date: Fri, 29 Nov 2024 15:20:03 +0100 Subject: [PATCH 3/3] minor cleaning --- web/src/SAP/src/services/search/transpiler.py | 33 +++---------------- 1 file changed, 4 insertions(+), 29 deletions(-) diff --git a/web/src/SAP/src/services/search/transpiler.py b/web/src/SAP/src/services/search/transpiler.py index 9945326..a3b08bc 100644 --- a/web/src/SAP/src/services/search/transpiler.py +++ b/web/src/SAP/src/services/search/transpiler.py @@ -82,47 +82,22 @@ def structure_wildcard(field, node): return {"$in": [coerced, node.term]} -def is_date_string(value): - try: - datetime.fromisoformat(value) - return True - except ValueError: - return False - -def is_float_string(value): - try: - float(value) - return True - except ValueError: - return False - -def convert_type(value): - if value.isdigit(): - return int(value) - elif is_float_string(value): - return float(value) - elif value.isdigit(): - return int(value) - elif is_date_string(value): - return datetime.fromisoformat(value) - return value - - def structure_ranged(field, node): min_op = "$gte" if node.inclusive == "left" or node.inclusive == "both" else "$gt" max_op = "$lte" if node.inclusive == "right" or node.inclusive == "both" else "$lt" + #if the hours, minutes and seconds are not in the search like this 2022-04-08T09:01:07 it is assumed that the entire day is intended to be included #default with the specific time of day not specified it is as if they are 00, which would exclude all records from during that day, which is not the behavior we expect is wanted - max_term = convert_type(node.term_max) + max_term = coerce_term(node.term_max) if type(max_term) == datetime and max_op =="$lte" and max_term.hour == 0 and max_term.minute == 0 and max_term.second == 0: max_term = max_term + timedelta(days = 1 ) - timedelta(seconds = 1) if node.term_min == "*": return {max_op: max_term} if node.term_max == "*": - return {min_op: convert_type(node.term_min)} + return {min_op: coerce_term(node.term_min)} - return {min_op: convert_type(node.term_min), max_op: max_term} + return {min_op: coerce_term(node.term_min), max_op: max_term} def structure_leaf(node, is_negated):