Merge branch 'master' into harshil/json-editor-revamp

startreedata · Oct 24, 2024 · 07e8f37 · 07e8f37
2 parents b89b6c6 + e4a504a
commit 07e8f37
Show file tree

Hide file tree

Showing 47 changed files with 353 additions and 92 deletions.
diff --git a/config/server.yaml b/config/server.yaml
@@ -106,13 +106,26 @@ rca:
     algorithm: simple
     params: {}
 
+## DEPRECATED - please use defaultWorkspaceConfiguration below
 time:
   # notification timezone
   timezone: UTC
   # notification time format pattern - see https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html 
   dateTimePattern: "MMM dd, yyyy HH:mm"
   # Limit onboarding alert replay back to JAN_1_2000_UTC
   minimumOnboardingStartTime: 946684800000
+
+defaultWorkspaceConfiguration:
+  timeConfiguration:
+    # notification timezone
+    timezone: UTC
+    # notification time format pattern - see https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html 
+    dateTimePattern: "MMM dd, yyyy HH:mm"
+    # Limit onboarding alert replay back to JAN_1_2000_UTC
+    minimumOnboardingStartTime: 946684800000
+  templateConfiguration:
+    # alert templates generate queries with a LIMIT statement - this is the default value of the limit statement
+    sqlLimitStatement: 100_000_001
 
 mockEvents:
   enabled: false

diff --git a/kubernetes/helm/startree-thirdeye/Chart.yaml b/kubernetes/helm/startree-thirdeye/Chart.yaml
@@ -16,7 +16,7 @@ apiVersion: v2
 appVersion: 0.5.0-SNAPSHOT # Should be maintained automatically by a pipeline
 name: startree-thirdeye
 description: One Stop Shop For Anomaly Detection.
-version: 2.42.0
+version: 2.43.0
 keywords:
   - thirdeye
   - metrics

diff --git a/kubernetes/helm/startree-thirdeye/templates/coordinator/configmap.yaml b/kubernetes/helm/startree-thirdeye/templates/coordinator/configmap.yaml
@@ -95,11 +95,14 @@ data:
       noTaskDelay: {{ .Values.worker.config.noTaskDelay | default 15 }}
       heartbeatInterval: {{ .Values.worker.config.heartbeatInterval | default 30 }}
       activeThresholdMultiplier: {{ .Values.worker.config.activeThresholdMultiplier | default 30 }}
-
-    time:
-      timezone: {{ .Values.time.timezone }}
-      dateTimePattern: "{{ .Values.time.dateTimePattern }}"
-      minimumOnboardingStartTime: {{ .Values.time.minimumOnboardingStartTime }}
+
+    defaultWorkspaceConfiguration:
+      timeConfiguration:
+        timezone: {{ .Values.defaultWorkspaceConfiguration.timeConfiguration.timezone }}
+        dateTimePattern: "{{ .Values.defaultWorkspaceConfiguration.timeConfiguration.dateTimePattern }}"
+        minimumOnboardingStartTime: {{ .Values.defaultWorkspaceConfiguration.timeConfiguration.minimumOnboardingStartTime }}
+      templateConfiguration:
+        sqlLimitStatement: {{ .Values.defaultWorkspaceConfiguration.templateConfiguration.sqlLimitStatement }}
 
     rca:
       topContributors:

diff --git a/kubernetes/helm/startree-thirdeye/templates/scheduler/scheduler-config.yaml b/kubernetes/helm/startree-thirdeye/templates/scheduler/scheduler-config.yaml
@@ -116,10 +116,13 @@ data:
       heartbeatInterval: {{ .Values.worker.config.heartbeatInterval | default 30 }}
       activeThresholdMultiplier: {{ .Values.worker.config.activeThresholdMultiplier | default 30 }}
     
-    time:
-      timezone: {{ .Values.time.timezone }}
-      dateTimePattern: "{{ .Values.time.dateTimePattern }}"
-      minimumOnboardingStartTime: {{ .Values.time.minimumOnboardingStartTime }}
+    defaultWorkspaceConfiguration:
+      timeConfiguration:
+        timezone: {{ .Values.defaultWorkspaceConfiguration.timeConfiguration.timezone }}
+        dateTimePattern: "{{ .Values.defaultWorkspaceConfiguration.timeConfiguration.dateTimePattern }}"
+        minimumOnboardingStartTime: {{ .Values.defaultWorkspaceConfiguration.timeConfiguration.minimumOnboardingStartTime }}
+      templateConfiguration:
+        sqlLimitStatement: {{ .Values.defaultWorkspaceConfiguration.templateConfiguration.sqlLimitStatement }}
     
     rca:
       topContributors:
@@ -206,5 +209,5 @@ data:
         enabled: false
         calendars: []
         {{- end }}
-
+        
 {{ end }}
diff --git a/kubernetes/helm/startree-thirdeye/templates/worker/worker-config.yaml b/kubernetes/helm/startree-thirdeye/templates/worker/worker-config.yaml
@@ -119,11 +119,14 @@ data:
       noTaskDelay: {{ .Values.worker.config.noTaskDelay | default 15 }}
       heartbeatInterval: {{ .Values.worker.config.heartbeatInterval | default 30 }}
       activeThresholdMultiplier: {{ .Values.worker.config.activeThresholdMultiplier | default 30 }}
-    
-    time:
-      timezone: {{ .Values.time.timezone }}
-      dateTimePattern: "{{ .Values.time.dateTimePattern }}"
-      minimumOnboardingStartTime: {{ .Values.time.minimumOnboardingStartTime }}
+      
+    defaultWorkspaceConfiguration:
+      timeConfiguration:
+        timezone: {{ .Values.defaultWorkspaceConfiguration.timeConfiguration.timezone }}
+        dateTimePattern: "{{ .Values.defaultWorkspaceConfiguration.timeConfiguration.dateTimePattern }}"
+        minimumOnboardingStartTime: {{ .Values.defaultWorkspaceConfiguration.timeConfiguration.minimumOnboardingStartTime }}
+      templateConfiguration:
+        sqlLimitStatement: {{ .Values.defaultWorkspaceConfiguration.templateConfiguration.sqlLimitStatement }}
     
     rca:
       topContributors:

diff --git a/kubernetes/helm/startree-thirdeye/values.yaml b/kubernetes/helm/startree-thirdeye/values.yaml
@@ -138,11 +138,15 @@ prometheus:
 sentry:
   dsn: ""
 
-time:
-  timezone: UTC
-  dateTimePattern: "MMM dd, yyyy HH:mm"
-  # JAN_1_2000_UTC
-  minimumOnboardingStartTime: 946684800000
+
+defaultWorkspaceConfiguration:
+  timeConfiguration:
+    timezone: UTC 
+    dateTimePattern: "MMM dd, yyyy HH:mm"
+    # JAN_1_2000_UTC
+    minimumOnboardingStartTime: 946684800000
+  templateConfiguration:
+    sqlLimitStatement: 100000000
 
 rca:
   topContributors:

diff --git a/pinot-test-container/pom.xml b/pinot-test-container/pom.xml
@@ -18,7 +18,7 @@
     <parent>
         <artifactId>thirdeye</artifactId>
         <groupId>ai.startree.thirdeye</groupId>
-        <version>1.345.0-SNAPSHOT</version>
+        <version>1.347.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/pom.xml b/pom.xml
@@ -18,7 +18,7 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>ai.startree.thirdeye</groupId>
   <artifactId>thirdeye</artifactId>
-  <version>1.345.0-SNAPSHOT</version>
+  <version>1.347.0-SNAPSHOT</version>
   <packaging>pom</packaging>
   <name>thirdeye</name>
 

diff --git a/thirdeye-benchmarks/pom.xml b/thirdeye-benchmarks/pom.xml
@@ -19,7 +19,7 @@
   <parent>
     <groupId>ai.startree.thirdeye</groupId>
     <artifactId>thirdeye</artifactId>
-    <version>1.345.0-SNAPSHOT</version>
+    <version>1.347.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>thirdeye-benchmarks</artifactId>

diff --git a/thirdeye-benchmarks/src/main/java/org/sample/LoadTemplatesBenchmark.java b/thirdeye-benchmarks/src/main/java/org/sample/LoadTemplatesBenchmark.java
@@ -16,6 +16,7 @@
 import ai.startree.thirdeye.plugins.bootstrap.opencore.OpenCoreBoostrapResourcesProvider;
 import ai.startree.thirdeye.spi.Constants;
 import ai.startree.thirdeye.spi.api.AlertTemplateApi;
+import ai.startree.thirdeye.spi.datalayer.dto.TemplateConfigurationDTO;
 import com.fasterxml.jackson.core.type.TypeReference;
 import java.io.IOException;
 import java.util.List;
@@ -49,22 +50,25 @@ public class LoadTemplatesBenchmark {
   int numNamespaces;
 
   OpenCoreBoostrapResourcesProvider openCoreBoostrapResourcesProvider;
+
+  TemplateConfigurationDTO templateConfiguration;
 
   @Setup
   public void setup() {
     openCoreBoostrapResourcesProvider = new OpenCoreBoostrapResourcesProvider();
+    templateConfiguration = new TemplateConfigurationDTO();
   }
 
   @Benchmark
   public void loadWithNoCache(Blackhole blackhole) throws IOException {
     for (int i = 0; i < numNamespaces; i++) {
-      blackhole.consume(openCoreBoostrapResourcesProvider.getAlertTemplates());
+      blackhole.consume(openCoreBoostrapResourcesProvider.getAlertTemplates(templateConfiguration));
     }
   }
 
   @Benchmark
   public void loadWithCacheAndCopy(Blackhole blackhole) throws IOException {
-    final List<AlertTemplateApi> alertTemplates = openCoreBoostrapResourcesProvider.getAlertTemplates();
+    final List<AlertTemplateApi> alertTemplates = openCoreBoostrapResourcesProvider.getAlertTemplates(templateConfiguration);
     for (int i = 0; i < numNamespaces; i++) {
       blackhole.consume(copy((alertTemplates)));
     }

diff --git a/thirdeye-core/pom.xml b/thirdeye-core/pom.xml
@@ -19,7 +19,7 @@
   <parent>
     <groupId>ai.startree.thirdeye</groupId>
     <artifactId>thirdeye</artifactId>
-    <version>1.345.0-SNAPSHOT</version>
+    <version>1.347.0-SNAPSHOT</version>
   </parent>
   <artifactId>thirdeye-core</artifactId>
 

diff --git a/thirdeye-core/src/main/java/ai/startree/thirdeye/ThirdEyeCoreModule.java b/thirdeye-core/src/main/java/ai/startree/thirdeye/ThirdEyeCoreModule.java
@@ -18,29 +18,40 @@
 import ai.startree.thirdeye.datasource.loader.DefaultAggregationLoader;
 import ai.startree.thirdeye.datasource.loader.DefaultMinMaxTimeLoader;
 import ai.startree.thirdeye.rootcause.configuration.RcaConfiguration;
+import ai.startree.thirdeye.spi.api.NamespaceConfigurationApi;
 import ai.startree.thirdeye.spi.config.TimeConfiguration;
+import ai.startree.thirdeye.spi.datalayer.dto.NamespaceConfigurationDTO;
 import ai.startree.thirdeye.spi.datasource.loader.AggregationLoader;
 import ai.startree.thirdeye.spi.datasource.loader.MinMaxTimeLoader;
 import com.google.inject.AbstractModule;
 import com.google.inject.Scopes;
+import com.google.inject.util.Providers;
+import java.security.Provider;
 import org.apache.tomcat.jdbc.pool.DataSource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class ThirdEyeCoreModule extends AbstractModule {
+
+  private static final Logger LOG = LoggerFactory.getLogger(ThirdEyeCoreModule.class);
 
   private final DataSource dataSource;
   private final RcaConfiguration rcaConfiguration;
   private final UiConfiguration uiConfiguration;
   private final TimeConfiguration timeConfiguration;
+  private final NamespaceConfigurationDTO defaultNamespaceConfiguration;
 
   public ThirdEyeCoreModule(final DataSource dataSource,
       final RcaConfiguration rcaConfiguration,
       final UiConfiguration uiConfiguration,
-      final TimeConfiguration timeConfiguration) {
+      final TimeConfiguration timeConfiguration,
+      final NamespaceConfigurationDTO defaultNamespaceConfiguration) {
     this.dataSource = dataSource;
 
     this.rcaConfiguration = rcaConfiguration;
     this.uiConfiguration = uiConfiguration;
     this.timeConfiguration = timeConfiguration;
+    this.defaultNamespaceConfiguration = defaultNamespaceConfiguration;
   }
 
   @Override
@@ -52,6 +63,12 @@ protected void configure() {
 
     bind(RcaConfiguration.class).toInstance(rcaConfiguration);
     bind(UiConfiguration.class).toInstance(uiConfiguration);
-    bind(TimeConfiguration.class).toInstance(timeConfiguration);
+    if (timeConfiguration != null) {
+      LOG.warn("Using the time configuration. This is deprecated. Please use defaultWorkspaceConfiguration.timeConfiguration instead.");
+      bind(TimeConfiguration.class).toInstance(timeConfiguration);
+    } else {
+      bind(TimeConfiguration.class).toProvider(Providers.of(null));
+    }
+    bind(NamespaceConfigurationDTO.class).toInstance(defaultNamespaceConfiguration);
   }
 }
diff --git a/thirdeye-dataframe/pom.xml b/thirdeye-dataframe/pom.xml
@@ -19,7 +19,7 @@
   <parent>
     <groupId>ai.startree.thirdeye</groupId>
     <artifactId>thirdeye</artifactId>
-    <version>1.345.0-SNAPSHOT</version>
+    <version>1.347.0-SNAPSHOT</version>
   </parent>
 
   <modelVersion>4.0.0</modelVersion>

diff --git a/thirdeye-detectionpipeline/pom.xml b/thirdeye-detectionpipeline/pom.xml
@@ -19,7 +19,7 @@
   <parent>
     <groupId>ai.startree.thirdeye</groupId>
     <artifactId>thirdeye</artifactId>
-    <version>1.345.0-SNAPSHOT</version>
+    <version>1.347.0-SNAPSHOT</version>
   </parent>
   <artifactId>thirdeye-detectionpipeline</artifactId>
 

diff --git a/thirdeye-distribution/pom.xml b/thirdeye-distribution/pom.xml
@@ -18,7 +18,7 @@
   <parent>
     <artifactId>thirdeye</artifactId>
     <groupId>ai.startree.thirdeye</groupId>
-    <version>1.345.0-SNAPSHOT</version>
+    <version>1.347.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

diff --git a/thirdeye-integration-tests/pom.xml b/thirdeye-integration-tests/pom.xml
@@ -18,7 +18,7 @@
   <parent>
     <artifactId>thirdeye</artifactId>
     <groupId>ai.startree.thirdeye</groupId>
-    <version>1.345.0-SNAPSHOT</version>
+    <version>1.347.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

diff --git a/thirdeye-notification/pom.xml b/thirdeye-notification/pom.xml
@@ -19,7 +19,7 @@
   <parent>
     <groupId>ai.startree.thirdeye</groupId>
     <artifactId>thirdeye</artifactId>
-    <version>1.345.0-SNAPSHOT</version>
+    <version>1.347.0-SNAPSHOT</version>
   </parent>
   <artifactId>thirdeye-notification</artifactId>
 

diff --git a/thirdeye-persistence/pom.xml b/thirdeye-persistence/pom.xml
@@ -18,7 +18,7 @@
   <parent>
     <artifactId>thirdeye</artifactId>
     <groupId>ai.startree.thirdeye</groupId>
-    <version>1.345.0-SNAPSHOT</version>
+    <version>1.347.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>