Merge branch 'main' into entitlements/file_check

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java

@@ -187,20 +187,12 @@ static String agentCommandLineOption(Path agentJar, Path tmpPropertiesFile) {
     static void extractSecureSettings(SecureSettings secrets, Map<String, String> propertiesMap) {
         final Set<String> settingNames = secrets.getSettingNames();
         for (String key : List.of("api_key", "secret_token")) {
-            for (String prefix : List.of("telemetry.", "tracing.apm.")) {
-                if (settingNames.contains(prefix + key)) {
-                    if (propertiesMap.containsKey(key)) {
-                        throw new IllegalStateException(
-                            Strings.format("Duplicate telemetry setting: [telemetry.%s] and [tracing.apm.%s]", key, key)
-                        );
-                    }
-
-                    try (SecureString token = secrets.getString(prefix + key)) {
-                        propertiesMap.put(key, token.toString());
-                    }
+            String prefix = "telemetry.";
+            if (settingNames.contains(prefix + key)) {
+                try (SecureString token = secrets.getString(prefix + key)) {
+                    propertiesMap.put(key, token.toString());
                 }
             }
-
         }
     }
 
@@ -227,44 +219,12 @@ private static Map<String, String> extractDynamicSettings(Map<String, String> pr
     static Map<String, String> extractApmSettings(Settings settings) throws UserException {
         final Map<String, String> propertiesMap = new HashMap<>();
 
-        // tracing.apm.agent. is deprecated by telemetry.agent.
         final String telemetryAgentPrefix = "telemetry.agent.";
-        final String deprecatedTelemetryAgentPrefix = "tracing.apm.agent.";
 
         final Settings telemetryAgentSettings = settings.getByPrefix(telemetryAgentPrefix);
         telemetryAgentSettings.keySet().forEach(key -> propertiesMap.put(key, String.valueOf(telemetryAgentSettings.get(key))));
 
-        final Settings apmAgentSettings = settings.getByPrefix(deprecatedTelemetryAgentPrefix);
-        for (String key : apmAgentSettings.keySet()) {
-            if (propertiesMap.containsKey(key)) {
-                throw new IllegalStateException(
-                    Strings.format(
-                        "Duplicate telemetry setting: [%s%s] and [%s%s]",
-                        telemetryAgentPrefix,
-                        key,
-                        deprecatedTelemetryAgentPrefix,
-                        key
-                    )
-                );
-            }
-            propertiesMap.put(key, String.valueOf(apmAgentSettings.get(key)));
-        }
-
         StringJoiner globalLabels = extractGlobalLabels(telemetryAgentPrefix, propertiesMap, settings);
-        if (globalLabels.length() == 0) {
-            globalLabels = extractGlobalLabels(deprecatedTelemetryAgentPrefix, propertiesMap, settings);
-        } else {
-            StringJoiner tracingGlobalLabels = extractGlobalLabels(deprecatedTelemetryAgentPrefix, propertiesMap, settings);
-            if (tracingGlobalLabels.length() != 0) {
-                throw new IllegalArgumentException(
-                    "Cannot have global labels with tracing.agent prefix ["
-                        + globalLabels
-                        + "] and telemetry.apm.agent prefix ["
-                        + tracingGlobalLabels
-                        + "]"
-                );
-            }
-        }
         if (globalLabels.length() > 0) {
             propertiesMap.put("global_labels", globalLabels.toString());
         }
@@ -274,7 +234,7 @@ static Map<String, String> extractApmSettings(Settings settings) throws UserExce
             if (propertiesMap.containsKey(key)) {
                 throw new UserException(
                     ExitCodes.CONFIG,
-                    "Do not set a value for [tracing.apm.agent." + key + "], as this is configured automatically by Elasticsearch"
+                    "Do not set a value for [telemetry.agent." + key + "], as this is configured automatically by Elasticsearch"
                 );
             }
         }

distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/APMJvmOptionsTests.java

@@ -25,18 +25,15 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.function.Function;
 
 import static org.elasticsearch.test.MapMatcher.matchesMap;
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.containsInAnyOrder;
-import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.hasSize;
-import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.mock;
@@ -82,109 +79,63 @@ public void testFileDeleteWorks() throws IOException {
     }
 
     public void testExtractSecureSettings() {
-        MockSecureSettings duplicateSecureSettings = new MockSecureSettings();
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("telemetry.secret_token", "token");
+        secureSettings.setString("telemetry.api_key", "key");
 
-        for (String prefix : List.of("telemetry.", "tracing.apm.")) {
-            MockSecureSettings secureSettings = new MockSecureSettings();
-            secureSettings.setString(prefix + "secret_token", "token");
-            secureSettings.setString(prefix + "api_key", "key");
-
-            duplicateSecureSettings.setString(prefix + "api_key", "secret");
-
-            Map<String, String> propertiesMap = new HashMap<>();
-            APMJvmOptions.extractSecureSettings(secureSettings, propertiesMap);
-
-            assertThat(propertiesMap, matchesMap(Map.of("secret_token", "token", "api_key", "key")));
-        }
-
-        Exception exception = expectThrows(
-            IllegalStateException.class,
-            () -> APMJvmOptions.extractSecureSettings(duplicateSecureSettings, new HashMap<>())
-        );
-        assertThat(exception.getMessage(), containsString("Duplicate telemetry setting"));
-        assertThat(exception.getMessage(), containsString("telemetry.api_key"));
-        assertThat(exception.getMessage(), containsString("tracing.apm.api_key"));
+        Map<String, String> propertiesMap = new HashMap<>();
+        APMJvmOptions.extractSecureSettings(secureSettings, propertiesMap);
 
+        assertThat(propertiesMap, matchesMap(Map.of("secret_token", "token", "api_key", "key")));
     }
 
     public void testExtractSettings() throws UserException {
-        Function<String, Settings.Builder> buildSettings = (prefix) -> Settings.builder()
-            .put(prefix + "server_url", "https://myurl:443")
-            .put(prefix + "service_node_name", "instance-0000000001");
-
-        for (String prefix : List.of("tracing.apm.agent.", "telemetry.agent.")) {
-            var name = "APM Tracing";
-            var deploy = "123";
-            var org = "456";
-            var extracted = APMJvmOptions.extractApmSettings(
-                buildSettings.apply(prefix)
-                    .put(prefix + "global_labels.deployment_name", name)
-                    .put(prefix + "global_labels.deployment_id", deploy)
-                    .put(prefix + "global_labels.organization_id", org)
-                    .build()
-            );
-
-            assertThat(
-                extracted,
-                allOf(
-                    hasEntry("server_url", "https://myurl:443"),
-                    hasEntry("service_node_name", "instance-0000000001"),
-                    hasEntry(equalTo("global_labels"), not(endsWith(","))), // test that we have collapsed all global labels into one
-                    not(hasKey("global_labels.organization_id")) // tests that we strip out the top level label keys
-                )
-            );
-
-            List<String> labels = Arrays.stream(extracted.get("global_labels").split(",")).toList();
-            assertThat(labels, hasSize(3));
-            assertThat(labels, containsInAnyOrder("deployment_name=APM Tracing", "organization_id=" + org, "deployment_id=" + deploy));
-
-            // test replacing with underscores and skipping empty
-            name = "APM=Tracing";
-            deploy = "";
-            org = ",456";
-            extracted = APMJvmOptions.extractApmSettings(
-                buildSettings.apply(prefix)
-                    .put(prefix + "global_labels.deployment_name", name)
-                    .put(prefix + "global_labels.deployment_id", deploy)
-                    .put(prefix + "global_labels.organization_id", org)
-                    .build()
-            );
-            labels = Arrays.stream(extracted.get("global_labels").split(",")).toList();
-            assertThat(labels, hasSize(2));
-            assertThat(labels, containsInAnyOrder("deployment_name=APM_Tracing", "organization_id=_456"));
-        }
-
-        IllegalStateException err = expectThrows(
-            IllegalStateException.class,
-            () -> APMJvmOptions.extractApmSettings(
-                Settings.builder()
-                    .put("tracing.apm.agent.server_url", "https://myurl:443")
-                    .put("telemetry.agent.server_url", "https://myurl-2:443")
-                    .build()
-            )
-        );
-        assertThat(err.getMessage(), is("Duplicate telemetry setting: [telemetry.agent.server_url] and [tracing.apm.agent.server_url]"));
-    }
-
-    public void testNoMixedLabels() {
-        String telemetryAgent = "telemetry.agent.";
-        String tracingAgent = "tracing.apm.agent.";
-        Settings settings = Settings.builder()
-            .put("tracing.apm.enabled", true)
-            .put(telemetryAgent + "server_url", "https://myurl:443")
-            .put(telemetryAgent + "service_node_name", "instance-0000000001")
-            .put(tracingAgent + "global_labels.deployment_id", "123")
-            .put(telemetryAgent + "global_labels.organization_id", "456")
+        Settings defaults = Settings.builder()
+            .put("telemetry.agent.server_url", "https://myurl:443")
+            .put("telemetry.agent.service_node_name", "instance-0000000001")
             .build();
 
-        IllegalArgumentException err = assertThrows(IllegalArgumentException.class, () -> APMJvmOptions.extractApmSettings(settings));
+        var name = "APM Tracing";
+        var deploy = "123";
+        var org = "456";
+        var extracted = APMJvmOptions.extractApmSettings(
+            Settings.builder()
+                .put(defaults)
+                .put("telemetry.agent.global_labels.deployment_name", name)
+                .put("telemetry.agent.global_labels.deployment_id", deploy)
+                .put("telemetry.agent.global_labels.organization_id", org)
+                .build()
+        );
+
         assertThat(
-            err.getMessage(),
-            is(
-                "Cannot have global labels with tracing.agent prefix [organization_id=456] and"
-                    + " telemetry.apm.agent prefix [deployment_id=123]"
+            extracted,
+            allOf(
+                hasEntry("server_url", "https://myurl:443"),
+                hasEntry("service_node_name", "instance-0000000001"),
+                hasEntry(equalTo("global_labels"), not(endsWith(","))), // test that we have collapsed all global labels into one
+                not(hasKey("global_labels.organization_id")) // tests that we strip out the top level label keys
             )
         );
+
+        List<String> labels = Arrays.stream(extracted.get("global_labels").split(",")).toList();
+        assertThat(labels, hasSize(3));
+        assertThat(labels, containsInAnyOrder("deployment_name=APM Tracing", "organization_id=" + org, "deployment_id=" + deploy));
+
+        // test replacing with underscores and skipping empty
+        name = "APM=Tracing";
+        deploy = "";
+        org = ",456";
+        extracted = APMJvmOptions.extractApmSettings(
+            Settings.builder()
+                .put(defaults)
+                .put("telemetry.agent.global_labels.deployment_name", name)
+                .put("telemetry.agent.global_labels.deployment_id", deploy)
+                .put("telemetry.agent.global_labels.organization_id", org)
+                .build()
+        );
+        labels = Arrays.stream(extracted.get("global_labels").split(",")).toList();
+        assertThat(labels, hasSize(2));
+        assertThat(labels, containsInAnyOrder("deployment_name=APM_Tracing", "organization_id=_456"));
     }
 
     private Path makeFakeAgentJar() throws IOException {

docs/changelog/119227.yaml

@@ -0,0 +1,13 @@
+pr: 119227
+summary: Remove unfreeze REST endpoint
+area: Indices APIs
+type: breaking
+issues: []
+breaking:
+  title: Remove unfreeze REST endpoint
+  area: REST API
+  details: >-
+    The `/{index}/_unfreeze` REST endpoint is no longer supported. This API was deprecated, and the corresponding
+    `/{index}/_freeze` endpoint was removed in 8.0.
+  impact: None, since it is not possible to have a frozen index in a version which is readable by Elasticsearch 9.0
+  notable: false

docs/changelog/119772.yaml

@@ -0,0 +1,6 @@
+pr: 119772
+summary: ESQL Support IN operator for Date nanos
+area: ES|QL
+type: enhancement
+issues:
+ - 118578

docs/changelog/119831.yaml

@@ -0,0 +1,5 @@
+pr: 119831
+summary: Run `TransportClusterGetSettingsAction` on local node
+area: Infra/Settings
+type: enhancement
+issues: []

docs/changelog/119846.yaml

@@ -0,0 +1,12 @@
+pr: 119846
+summary: Drop support for brackets from METADATA syntax
+area: ES|QL
+type: deprecation
+issues:
+ - 115401
+deprecation:
+  title: Drop support for brackets from METADATA syntax
+  area: ES|QL
+  details: Please describe the details of this change for the release notes. You can
+    use asciidoc.
+  impact: Please describe the impact of this change to users

docs/changelog/119922.yaml

@@ -0,0 +1,5 @@
+pr: 119922
+summary: "[Inference API] fix spell words: covertToString to convertToString"
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/119926.yaml

@@ -0,0 +1,11 @@
+pr: 119926
+summary: "Deprecated tracing.apm.* settings got removed."
+area: Infra/Metrics
+type: breaking
+issues: []
+breaking:
+  title: "Deprecated tracing.apm.* settings got removed."
+  area: Cluster and node setting
+  details: Deprecated `tracing.apm.*` settings got removed, use respective `telemetry.*` / `telemetry.tracing.*` settings instead.
+  impact: 9.x nodes will refuse to start if any such setting (including secret settings) is still present.
+  notable: false

docs/changelog/120014.yaml

@@ -0,0 +1,6 @@
+pr: 120014
+summary: Fix potential file leak in ES816BinaryQuantizedVectorsWriter
+area: Search
+type: bug
+issues:
+ - 119981

docs/changelog/120038.yaml

@@ -0,0 +1,5 @@
+pr: 120038
+summary: Run template simulation actions on local node
+area: Ingest Node
+type: enhancement
+issues: []

docs/changelog/120084.yaml

@@ -0,0 +1,5 @@
+pr: 120084
+summary: Improve how reindex data stream index action handles api blocks
+area: Data streams
+type: enhancement
+issues: []

docs/changelog/120133.yaml

@@ -0,0 +1,6 @@
+pr: 120133
+summary: Use approximation to advance matched queries
+area: Search
+type: bug
+issues:
+ - 120130

docs/reference/esql/esql-limitations.asciidoc

@@ -30,11 +30,11 @@ include::processing-commands/limit.asciidoc[tag=limitation]
 ** You can use `to_datetime` to cast to millisecond dates to use unsupported functions
 * `double` (`float`, `half_float`, `scaled_float` are represented as `double`)
 * `ip`
-* `keyword` family including `keyword`, `constant_keyword`, and `wildcard`
+* `keyword` <<keyword, family>> including `keyword`, `constant_keyword`, and `wildcard`
 * `int` (`short` and `byte` are represented as `int`)
 * `long`
 * `null`
-* `text`
+* `text` <<text, family>> including `text`, `semantic_text` and `match_only_text`
 * experimental:[] `unsigned_long`
 * `version`
 * Spatial types

docs/reference/indices.asciidoc

@@ -24,7 +24,6 @@ index settings, aliases, mappings, and index templates.
 * <<indices-split-index>>
 * <<indices-clone-index>>
 * <<indices-rollover-index>>
-* <<unfreeze-index-api>>
 * <<indices-resolve-index-api>>
 * <<indices-resolve-cluster-api>>
 * <<indices-downsample-data-stream>>
@@ -143,6 +142,5 @@ include::indices/shrink-index.asciidoc[]
 include::indices/simulate-index.asciidoc[]
 include::indices/simulate-template.asciidoc[]
 include::indices/split-index.asciidoc[]
-include::indices/apis/unfreeze.asciidoc[]
 include::indices/update-settings.asciidoc[]
 include::indices/put-mapping.asciidoc[]