Add missing traces ilm policy for OTel traces data streams (#119449) (#…

…119825)
elastic · Jan 9, 2025 · 7324f31 · 7324f31
1 parent 0522df0
commit 7324f31
Show file tree

Hide file tree

Showing 7 changed files with 52 additions and 3 deletions.
diff --git a/docs/changelog/119449.yaml b/docs/changelog/119449.yaml
@@ -0,0 +1,5 @@
+pr: 119449
+summary: Add missing traces ilm policy for OTel traces data streams
+area: Data streams
+type: bug
+issues: []
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
@@ -801,6 +801,7 @@ protected Set<String> preserveILMPolicyIds() {
             "profiling-60-days@lifecycle",
             "synthetics",
             "synthetics@lifecycle",
+            "traces@lifecycle",
             "7-days-default",
             "7-days@lifecycle",
             "30-days-default",

diff --git a/x-pack/plugin/core/template-resources/src/main/resources/[email protected] b/x-pack/plugin/core/template-resources/src/main/resources/[email protected]
@@ -0,0 +1,17 @@
+{
+  "phases": {
+    "hot": {
+      "actions": {
+        "rollover": {
+          "max_primary_shard_size": "50gb",
+          "max_age": "30d"
+        }
+      }
+    }
+  },
+  "_meta": {
+    "description": "default policy for the traces index template installed by x-pack",
+    "managed": true
+  },
+  "deprecated": ${xpack.stack.template.deprecated}
+}
diff --git a/x-pack/plugin/core/template-resources/src/main/resources/[email protected] b/x-pack/plugin/core/template-resources/src/main/resources/[email protected]
@@ -2,6 +2,9 @@
   "template": {
     "settings": {
       "index": {
+        "lifecycle": {
+          "name": "traces@lifecycle"
+        },
         "codec": "best_compression",
         "mapping": {
           "ignore_malformed": true

diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml
@@ -120,3 +120,23 @@ Conflicting attribute types:
   - is_true: $datastream-backing-index
   - match: { .$datastream-backing-index.settings.index.sort.field.0: "resource.attributes.host.name" }
   - match: { .$datastream-backing-index.settings.index.sort.field.1: "@timestamp" }
+
+---
+traces@lifecycle:
+  - do:
+      bulk:
+        index: traces-generic.otel-default
+        refresh: true
+        body:
+          - create: {}
+          - '{"@timestamp":"2024-07-18T14:49:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default"}, "span_id":"1"}'
+  - is_false: errors
+  - do:
+      indices.get_data_stream:
+        name: traces-generic.otel-default
+  - set: { data_streams.0.indices.0.index_name: datastream-backing-index }
+  - do:
+      indices.get_settings:
+        index: $datastream-backing-index
+  - is_true: $datastream-backing-index
+  - match: { .$datastream-backing-index.settings.index.lifecycle.name: "traces@lifecycle" }
diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java
@@ -101,6 +101,7 @@ public class StackTemplateRegistry extends IndexTemplateRegistry {
     //////////////////////////////////////////////////////////
     public static final String TRACES_MAPPINGS_COMPONENT_TEMPLATE_NAME = "traces@mappings";
     public static final String TRACES_SETTINGS_COMPONENT_TEMPLATE_NAME = "traces@settings";
+    public static final String TRACES_ILM_POLICY_NAME = "traces@lifecycle";
 
     //////////////////////////////////////////////////////////
     // Synthetics components (for matching synthetics-*-* indices)
@@ -254,6 +255,7 @@ private void updateEnabledSetting(boolean newValue) {
         new LifecyclePolicyConfig(LOGS_ILM_POLICY_NAME, "/[email protected]", ADDITIONAL_TEMPLATE_VARIABLES),
         new LifecyclePolicyConfig(METRICS_ILM_POLICY_NAME, "/[email protected]", ADDITIONAL_TEMPLATE_VARIABLES),
         new LifecyclePolicyConfig(SYNTHETICS_ILM_POLICY_NAME, "/[email protected]", ADDITIONAL_TEMPLATE_VARIABLES),
+        new LifecyclePolicyConfig(TRACES_ILM_POLICY_NAME, "/[email protected]", ADDITIONAL_TEMPLATE_VARIABLES),
         new LifecyclePolicyConfig(ILM_7_DAYS_POLICY_NAME, "/[email protected]", ADDITIONAL_TEMPLATE_VARIABLES),
         new LifecyclePolicyConfig(ILM_30_DAYS_POLICY_NAME, "/[email protected]", ADDITIONAL_TEMPLATE_VARIABLES),
         new LifecyclePolicyConfig(ILM_90_DAYS_POLICY_NAME, "/[email protected]", ADDITIONAL_TEMPLATE_VARIABLES),

diff --git a/.../plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java b/.../plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java
@@ -189,6 +189,7 @@ public void testThatNonExistingPoliciesAreAddedImmediately() throws Exception {
                         equalTo(StackTemplateRegistry.LOGS_ILM_POLICY_NAME),
                         equalTo(StackTemplateRegistry.METRICS_ILM_POLICY_NAME),
                         equalTo(StackTemplateRegistry.SYNTHETICS_ILM_POLICY_NAME),
+                        equalTo(StackTemplateRegistry.TRACES_ILM_POLICY_NAME),
                         equalTo(StackTemplateRegistry.ILM_7_DAYS_POLICY_NAME),
                         equalTo(StackTemplateRegistry.ILM_30_DAYS_POLICY_NAME),
                         equalTo(StackTemplateRegistry.ILM_90_DAYS_POLICY_NAME),
@@ -212,7 +213,7 @@ public void testThatNonExistingPoliciesAreAddedImmediately() throws Exception {
 
         ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), nodes);
         registry.clusterChanged(event);
-        assertBusy(() -> assertThat(calledTimes.get(), equalTo(8)));
+        assertBusy(() -> assertThat(calledTimes.get(), equalTo(9)));
     }
 
     public void testPolicyAlreadyExists() {
@@ -221,7 +222,7 @@ public void testPolicyAlreadyExists() {
 
         Map<String, LifecyclePolicy> policyMap = new HashMap<>();
         List<LifecyclePolicy> policies = registry.getLifecyclePolicies();
-        assertThat(policies, hasSize(8));
+        assertThat(policies, hasSize(9));
         policies.forEach(p -> policyMap.put(p.getName(), p));
 
         client.setVerifier((action, request, listener) -> {
@@ -292,7 +293,7 @@ public void testPolicyAlreadyExistsButDiffers() throws IOException {
         Map<String, LifecyclePolicy> policyMap = new HashMap<>();
         String policyStr = "{\"phases\":{\"delete\":{\"min_age\":\"1m\",\"actions\":{\"delete\":{}}}}}";
         List<LifecyclePolicy> policies = registry.getLifecyclePolicies();
-        assertThat(policies, hasSize(8));
+        assertThat(policies, hasSize(9));
         policies.forEach(p -> policyMap.put(p.getName(), p));
 
         client.setVerifier((action, request, listener) -> {