Merge pull request #123 from ssi-dk/skp/mock-tbr

Improvements to search functionality

bifrost/bifrost_db/initdb.d/tbr.generated.jsonl

@@ -1,3 +1,3 @@
-{"isolate_id":"1","institution":"SSI","date_received":"1945-12-13T12:05:56.143Z","run_id":"culpa tempor","provided_species":"culpa officia sint id","primary_isolate":false,"date_run":"1978-01-12T02:11:46.207Z","project_number":-27841476.650654063,"project_title":"nisi officia","date_sample":"1973-11-09T21:59:05.017Z","public":"amet ut eu sit deserunt","cpr_nr":"2902007734","gender":"K","name":"Elliot Rosenbaum","age":53,"travel":"et occaecat Lorem adipisicing","travel_country":"Guernsey","date_received_kma":"1947-08-02T21:01:12.522Z","kma":"ul","region":"sint cupidatat veniam","fud_number":"nostrud occaecat","cluster_id":"cillum nostrud cupidatat non officia","epi_export":"consequat enim exercitation"}
-{"isolate_id":"2","institution":"SSI","date_received":"1994-11-20T22:49:04.27Z","run_id":"laboris tempor","provided_species":"eu culpa exercitation","primary_isolate":true,"date_run":"1945-01-15T16:52:56.924Z","project_number":-33496873.684678532,"project_title":"labore","date_sample":"2003-11-26T18:48:06.257Z","public":"dolore aute consectetur","cpr_nr":"0000000002","gender":"M","name":"Adaline Waelchi","age":0,"travel":"Duis","travel_country":"Serbia","date_received_kma":"1971-03-15T05:10:15.052Z","kma":"ea","region":"cupidatat ex","fud_number":"non","cluster_id":"tempor i","epi_export":"dolor aute"}
-{"isolate_id":"3","institution":"SSI","date_received":"1987-05-15T16:41:34.514Z","run_id":"in Excepteur nostrud proident","provided_species":"aute minim","primary_isolate":false,"date_run":"1980-01-04T05:31:12.256Z","project_number":12997890.419586033,"project_title":"consectetur esse","date_sample":"1946-03-26T02:37:54.67Z","public":"pariatur","cpr_nr":"0000000001","gender":"K","name":"Brendon Ledner","age":6,"travel":"incididunt est veniam","travel_country":"Ethiopia","date_received_kma":"1987-07-10T12:45:08.443Z","kma":"nostrud aliquip velit dolor Lore","region":"sint et","fud_number":"enim nulla ut","cluster_id":"do Ut incididunt magna nisi","epi_export":"laboris mollit"}
+{"isolate_id":"2010F00006","institution":"SSI","date_received":"1945-12-13T12:05:56.143Z","run_id":"culpa tempor","provided_species":"culpa officia sint id","primary_isolate":false,"date_run":"1978-01-12T02:11:46.207Z","project_number":-27841476.650654063,"project_title":"nisi officia","date_sample":"1973-11-09T21:59:05.017Z","public":"amet ut eu sit deserunt","cpr_nr":"2902007734","gender":"K","name":"Elliot Rosenbaum","age":53,"travel":"et occaecat Lorem adipisicing","travel_country":"Guernsey","date_received_kma":"1947-08-02T21:01:12.522Z","kma":"ul","region":"sint cupidatat veniam","fud_number":"nostrud occaecat","cluster_id":"cillum nostrud cupidatat non officia","epi_export":"consequat enim exercitation"}
+{"isolate_id":"2010F00048","institution":"SSI","date_received":"1994-11-20T22:49:04.27Z","run_id":"laboris tempor","provided_species":"eu culpa exercitation","primary_isolate":true,"date_run":"1945-01-15T16:52:56.924Z","project_number":-33496873.684678532,"project_title":"labore","date_sample":"2003-11-26T18:48:06.257Z","public":"dolore aute consectetur","cpr_nr":"0000000002","gender":"M","name":"Adaline Waelchi","age":0,"travel":"Duis","travel_country":"Serbia","date_received_kma":"1971-03-15T05:10:15.052Z","kma":"ea","region":"cupidatat ex","fud_number":"non","cluster_id":"tempor i","epi_export":"dolor aute"}
+{"isolate_id":"2009T00243","institution":"SSI","date_received":"1987-05-15T16:41:34.514Z","run_id":"in Excepteur nostrud proident","provided_species":"aute minim","primary_isolate":false,"date_run":"1980-01-04T05:31:12.256Z","project_number":12997890.419586033,"project_title":"consectetur esse","date_sample":"1946-03-26T02:37:54.67Z","public":"pariatur","cpr_nr":"0000000001","gender":"K","name":"Brendon Ledner","age":6,"travel":"incididunt est veniam","travel_country":"Ethiopia","date_received_kma":"1987-07-10T12:45:08.443Z","kma":"nostrud aliquip velit dolor Lore","region":"sint et","fud_number":"enim nulla ut","cluster_id":"do Ut incididunt magna nisi","epi_export":"laboris mollit"}

sap_tbr_integration/DG.SAP.TBRIntegration/DG.SAP.TBRIntegration.csproj

@@ -18,5 +18,11 @@
     <PackageReference Include="System.Data.SqlClient" Version="4.8.2" />
   </ItemGroup>
 
+  <ItemGroup>
+    <None Update="tbr.generated.jsonl">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
 
 </Project>

sap_tbr_integration/DG.SAP.TBRIntegration/Program.cs

@@ -26,10 +26,14 @@ public static IHostBuilder CreateHostBuilder(string[] args) =>
                 {
                     logging.ClearProviders();
                     logging.AddConfiguration(hosting.Configuration.GetSection("Logging"));
-                    logging.AddEventLog(new EventLogSettings()
-                    {
-                        SourceName = "TBR.SAP.Integration",                        
-                    });
+                    var isMock = hosting.HostingEnvironment.IsDevelopment();
+                    Console.WriteLine($"IsMock: {isMock}");
+                    if (!isMock) {
+                        logging.AddEventLog(new EventLogSettings()
+                        {
+                            SourceName = "TBR.SAP.Integration",
+                        });
+                    }                    
                     logging.AddConsole();
                 });
     }

sap_tbr_integration/DG.SAP.TBRIntegration/Repositories/MockIsolateRepository.cs

@@ -4,25 +4,23 @@
 using System.Linq;
 using System.Threading.Tasks;
 using DG.SAP.TBRIntegration.Models;
+using System.IO;
+using Newtonsoft.Json;
 
 namespace DG.SAP.TBRIntegration.Repositories
 {
     public class MockIsolateRepository : IIsolateRepository
     {
-        private readonly List<Isolate> _isolates = new List<Isolate> {
-            new Isolate
-            {
-                IsolateId = "Test1"
-            },
-            new Isolate
-            {
-                IsolateId = "Test2"
-            },
-            new Isolate
-            {
-                IsolateId = "Test3"
-            }
-        };
+        private readonly Lazy<List<Isolate>> _isolates = new Lazy<List<Isolate>>(() =>
+        {
+            var lines = File.ReadAllLines("tbr.generated.jsonl");
+
+            if (lines.Count() == 0)
+                throw new Exception("tbr.generated.jsonl empty or not found");
+
+            return lines.Select(l => JsonConvert.DeserializeObject<Isolate>(l.Replace("_", ""))).ToList();
+        });
+
 
         public Task<bool> UpdateIsolate(IsolateUpdate isolateUpdate)
         {
@@ -31,7 +29,7 @@ public Task<bool> UpdateIsolate(IsolateUpdate isolateUpdate)
 
         public Task<Isolate> GetIsolate(string isolateId)
         {
-            return Task.FromResult(_isolates.FirstOrDefault(i => i.IsolateId == isolateId));
+            return Task.FromResult(_isolates.Value.FirstOrDefault(i => i.IsolateId == isolateId));
         }
 
         public Task<IList<RowVersion>> GetChangedIsolateIds(IList<RowVersion> isolates)

sap_tbr_integration/DG.SAP.TBRIntegration/Startup.cs

@@ -19,9 +19,12 @@ namespace DG.SAP.TBRIntegration
 {
     public class Startup
     {
-        public Startup(IConfiguration configuration)
+        private readonly IHostEnvironment hostEnvironment;
+
+        public Startup(IConfiguration configuration, IHostEnvironment hostEnvironment)
         {
             Configuration = configuration;
+            this.hostEnvironment = hostEnvironment;
         }
 
         public IConfiguration Configuration { get; }
@@ -41,7 +44,14 @@ public void ConfigureServices(IServiceCollection services)
                 });
             });
 
-            LoadDependencies(services);
+            if (hostEnvironment.IsDevelopment())
+            {
+                LoadMockDependencies(services);
+            }
+            else
+            {
+                LoadDependencies(services);
+            }
             services.AddTransient<IIsolateChangeService, IsolateChangeService>();
         }
 

sap_tbr_integration/DG.SAP.TBRIntegration/tbr.generated.jsonl

@@ -0,0 +1,3 @@
+{"isolate_id":"2010F00006","institution":"SSI","date_received":"1945-12-13T12:05:56.143Z","run_id":"culpa tempor","provided_species":"culpa officia sint id","primary_isolate":false,"date_run":"1978-01-12T02:11:46.207Z","project_number":-27841476.650654063,"project_title":"nisi officia","date_sample":"1973-11-09T21:59:05.017Z","public":"amet ut eu sit deserunt","cpr_nr":"2902007734","gender":"K","name":"Elliot Rosenbaum","age":53,"travel":"et occaecat Lorem adipisicing","travel_country":"Guernsey","date_received_kma":"1947-08-02T21:01:12.522Z","kma":"ul","region":"sint cupidatat veniam","fud_number":"nostrud occaecat","cluster_id":"cillum nostrud cupidatat non officia","epi_export":"consequat enim exercitation"}
+{"isolate_id":"2010F00048","institution":"SSI","date_received":"1994-11-20T22:49:04.27Z","run_id":"laboris tempor","provided_species":"eu culpa exercitation","primary_isolate":true,"date_run":"1945-01-15T16:52:56.924Z","project_number":-33496873.684678532,"project_title":"labore","date_sample":"2003-11-26T18:48:06.257Z","public":"dolore aute consectetur","cpr_nr":"0000000002","gender":"M","name":"Adaline Waelchi","age":0,"travel":"Duis","travel_country":"Serbia","date_received_kma":"1971-03-15T05:10:15.052Z","kma":"ea","region":"cupidatat ex","fud_number":"non","cluster_id":"tempor i","epi_export":"dolor aute"}
+{"isolate_id":"2009T00243","institution":"SSI","date_received":"1987-05-15T16:41:34.514Z","run_id":"in Excepteur nostrud proident","provided_species":"aute minim","primary_isolate":false,"date_run":"1980-01-04T05:31:12.256Z","project_number":12997890.419586033,"project_title":"consectetur esse","date_sample":"1946-03-26T02:37:54.67Z","public":"pariatur","cpr_nr":"0000000001","gender":"K","name":"Brendon Ledner","age":6,"travel":"incididunt est veniam","travel_country":"Ethiopia","date_received_kma":"1987-07-10T12:45:08.443Z","kma":"nostrud aliquip velit dolor Lore","region":"sint et","fud_number":"enim nulla ut","cluster_id":"do Ut incididunt magna nisi","epi_export":"laboris mollit"}

web/src/SAP/src/services/search/transpiler.py

@@ -69,6 +69,10 @@ def coerce_term(term: str):
 
 
 def structure_wildcard(field, node):
+    if field in pii_columns():
+        if field == "cpr_nr":   # because cpr is saved as a string in the DB, so don't convert even when consists of only digits
+            return node.term
+        return coerce_term(node.term)
     coerced = coerce_term(node.term)
     if isinstance(coerced, str):
         return check_for_wildcard(field, coerced)
@@ -78,45 +82,22 @@ def structure_wildcard(field, node):
         return {"$in": [coerced, node.term]}
 
 
-def is_date_string(value):
-    try:
-        datetime.fromisoformat(value)
-        return True
-    except ValueError:
-        return False
-
-def is_float_string(value):
-    try:
-        float(value)
-        return True
-    except ValueError:
-        return False
-
-def convert_type(value):
-    if is_float_string(value):
-        return float(value)
-    elif value.isdigit():
-        return int(value)
-    elif is_date_string(value):
-        return datetime.fromisoformat(value)
-    return value
-
-
 def structure_ranged(field, node):
     min_op = "$gte" if node.inclusive == "left" or node.inclusive == "both" else "$gt"
     max_op = "$lte" if node.inclusive == "right" or node.inclusive == "both" else "$lt"
-    if node.term_min == "*":
-        return {max_op: convert_type(node.term_max)}
-    if node.term_max == "*":
-        return {min_op: convert_type(node.term_min)}
 
     #if the hours, minutes and seconds are not in the search like this 2022-04-08T09:01:07 it is assumed that the entire day is intended to be included
-    #default with the specific time of day not specified it is as if they are 00, which would exclude all records from during that day, which is not the behavior we expect is wanted
-    max_term = convert_type(node.term_max) 
+    #default with the specific time of day not specified it is as if they are 00, which would exclude all records from during that day, which is not the behavior we expect is wanted   
+    max_term = coerce_term(node.term_max)
     if type(max_term) == datetime and max_op =="$lte" and max_term.hour == 0 and max_term.minute == 0 and max_term.second == 0:
         max_term = max_term + timedelta(days = 1 ) - timedelta(seconds = 1)
 
-    return {min_op: convert_type(node.term_min), max_op: max_term}
+    if node.term_min == "*":
+        return {max_op: max_term}
+    if node.term_max == "*":
+        return {min_op: coerce_term(node.term_min)}
+
+    return {min_op: coerce_term(node.term_min), max_op: max_term}
 
 
 def structure_leaf(node, is_negated):