Merge branch 'main' into shutdown/ephemeral_id

rjernst · Dec 27, 2024 · 8671329 · 8671329
2 parents 171d212 + b790628
commit 8671329
Show file tree

Hide file tree

Showing 287 changed files with 4,921 additions and 1,263 deletions.
diff --git a/...rc/main/java/org/elasticsearch/benchmark/index/mapper/LogsDbDocumentParsingBenchmark.java b/...rc/main/java/org/elasticsearch/benchmark/index/mapper/LogsDbDocumentParsingBenchmark.java
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/vector/VectorScorerBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/vector/VectorScorerBenchmark.java
@@ -83,6 +83,8 @@ public class VectorScorerBenchmark {
 
     RandomVectorScorer luceneDotScorerQuery;
     RandomVectorScorer nativeDotScorerQuery;
+    RandomVectorScorer luceneSqrScorerQuery;
+    RandomVectorScorer nativeSqrScorerQuery;
 
     @Setup
     public void setup() throws IOException {
@@ -130,6 +132,8 @@ public void setup() throws IOException {
         }
         luceneDotScorerQuery = luceneScorer(values, VectorSimilarityFunction.DOT_PRODUCT, queryVec);
         nativeDotScorerQuery = factory.getInt7SQVectorScorer(VectorSimilarityFunction.DOT_PRODUCT, values, queryVec).get();
+        luceneSqrScorerQuery = luceneScorer(values, VectorSimilarityFunction.EUCLIDEAN, queryVec);
+        nativeSqrScorerQuery = factory.getInt7SQVectorScorer(VectorSimilarityFunction.EUCLIDEAN, values, queryVec).get();
 
         // sanity
         var f1 = dotProductLucene();
@@ -157,6 +161,12 @@ public void setup() throws IOException {
         if (q1 != q2) {
             throw new AssertionError("query: lucene[" + q1 + "] != " + "native[" + q2 + "]");
         }
+
+        var sqr1 = squareDistanceLuceneQuery();
+        var sqr2 = squareDistanceNativeQuery();
+        if (sqr1 != sqr2) {
+            throw new AssertionError("query: lucene[" + q1 + "] != " + "native[" + q2 + "]");
+        }
     }
 
     @TearDown
@@ -217,6 +227,16 @@ public float squareDistanceScalar() {
         return 1 / (1f + adjustedDistance);
     }
 
+    @Benchmark
+    public float squareDistanceLuceneQuery() throws IOException {
+        return luceneSqrScorerQuery.score(1);
+    }
+
+    @Benchmark
+    public float squareDistanceNativeQuery() throws IOException {
+        return nativeSqrScorerQuery.score(1);
+    }
+
     QuantizedByteVectorValues vectorValues(int dims, int size, IndexInput in, VectorSimilarityFunction sim) throws IOException {
         var sq = new ScalarQuantizer(0.1f, 0.9f, (byte) 7);
         var slice = in.slice("values", 0, in.length());

diff --git a/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle b/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle
@@ -98,8 +98,8 @@ develocity {
           link 'Source', "${prBaseUrl}/tree/${System.getenv('BUILDKITE_COMMIT')}"
           link 'Pull Request', "https://github.com/${repository}/pull/${prId}"
         } else {
-          value 'Git Commit ID', gitRevision
-          link 'Source', "https://github.com/${repository}/tree/${gitRevision}"
+          value 'Git Commit ID', gitRevision.get()
+          link 'Source', "https://github.com/${repository}/tree/${gitRevision.get()}"
         }
 
         buildFinished { result ->

diff --git a/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle b/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle
@@ -166,7 +166,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
   tasks.register('buildDependencyArtifacts') {
     group = 'ide'
     description = 'Builds artifacts needed as dependency for IDE modules'
-    dependsOn([':plugins:repository-hdfs:hadoop-client-api:shadowJar',
+    dependsOn([':plugins:repository-hdfs:hadoop-client-api:jar',
       ':x-pack:plugin:esql:compute:ann:jar',
       ':x-pack:plugin:esql:compute:gen:jar',
       ':server:generateModulesList',

diff --git a/...rg/elasticsearch/gradle/internal/distribution/InternalElasticsearchDistributionTypes.java b/...rg/elasticsearch/gradle/internal/distribution/InternalElasticsearchDistributionTypes.java
@@ -14,14 +14,14 @@
 import java.util.List;
 
 public class InternalElasticsearchDistributionTypes {
-    public static ElasticsearchDistributionType DEB = new DebElasticsearchDistributionType();
-    public static ElasticsearchDistributionType RPM = new RpmElasticsearchDistributionType();
-    public static ElasticsearchDistributionType DOCKER = new DockerElasticsearchDistributionType();
-    public static ElasticsearchDistributionType DOCKER_IRONBANK = new DockerIronBankElasticsearchDistributionType();
-    public static ElasticsearchDistributionType DOCKER_CLOUD_ESS = new DockerCloudEssElasticsearchDistributionType();
-    public static ElasticsearchDistributionType DOCKER_WOLFI = new DockerWolfiElasticsearchDistributionType();
+    public static final ElasticsearchDistributionType DEB = new DebElasticsearchDistributionType();
+    public static final ElasticsearchDistributionType RPM = new RpmElasticsearchDistributionType();
+    public static final ElasticsearchDistributionType DOCKER = new DockerElasticsearchDistributionType();
+    public static final ElasticsearchDistributionType DOCKER_IRONBANK = new DockerIronBankElasticsearchDistributionType();
+    public static final ElasticsearchDistributionType DOCKER_CLOUD_ESS = new DockerCloudEssElasticsearchDistributionType();
+    public static final ElasticsearchDistributionType DOCKER_WOLFI = new DockerWolfiElasticsearchDistributionType();
 
-    public static List<ElasticsearchDistributionType> ALL_INTERNAL = List.of(
+    public static final List<ElasticsearchDistributionType> ALL_INTERNAL = List.of(
         DEB,
         RPM,
         DOCKER,

diff --git a/...al/src/main/java/org/elasticsearch/gradle/internal/test/rerun/TestRerunTaskExtension.java b/...al/src/main/java/org/elasticsearch/gradle/internal/test/rerun/TestRerunTaskExtension.java
@@ -30,7 +30,7 @@ public class TestRerunTaskExtension {
     /**
      * The name of the extension added to each test task.
      */
-    public static String NAME = "rerun";
+    public static final String NAME = "rerun";
 
     private final Property<Integer> maxReruns;
 

diff --git a/...src/test/groovy/org/elasticsearch/gradle/internal/info/BuildParameterExtensionSpec.groovy b/...src/test/groovy/org/elasticsearch/gradle/internal/info/BuildParameterExtensionSpec.groovy
@@ -9,9 +9,9 @@
 
 package org.elasticsearch.gradle.internal.info
 
+import spock.lang.Ignore
 import spock.lang.Specification
 
-import org.elasticsearch.gradle.internal.BwcVersions
 import org.gradle.api.JavaVersion
 import org.gradle.api.Project
 import org.gradle.api.provider.Provider
@@ -31,6 +31,7 @@ class BuildParameterExtensionSpec extends Specification {
 
     ProjectBuilder projectBuilder = new ProjectBuilder()
 
+    @Ignore
     def "#getterName is cached anc concurrently accessible"() {
         given:
         def project = projectBuilder.build()

diff --git a/...s/src/main/java/org/elasticsearch/gradle/distribution/ElasticsearchDistributionTypes.java b/...s/src/main/java/org/elasticsearch/gradle/distribution/ElasticsearchDistributionTypes.java
@@ -12,6 +12,6 @@
 import org.elasticsearch.gradle.ElasticsearchDistributionType;
 
 public class ElasticsearchDistributionTypes {
-    public static ElasticsearchDistributionType ARCHIVE = new ArchiveElasticsearchDistributionType();
-    public static ElasticsearchDistributionType INTEG_TEST_ZIP = new IntegTestZipElasticsearchDistributionType();
+    public static final ElasticsearchDistributionType ARCHIVE = new ArchiveElasticsearchDistributionType();
+    public static final ElasticsearchDistributionType INTEG_TEST_ZIP = new IntegTestZipElasticsearchDistributionType();
 }
diff --git a/distribution/docker/src/docker/Dockerfile b/distribution/docker/src/docker/Dockerfile
@@ -22,7 +22,7 @@
 <% if (docker_base == 'iron_bank') { %>
 ARG BASE_REGISTRY=registry1.dso.mil
 ARG BASE_IMAGE=ironbank/redhat/ubi/ubi9
-ARG BASE_TAG=9.4
+ARG BASE_TAG=9.5
 <% } %>
 
 ################################################################################

diff --git a/docs/changelog/116868.yaml b/docs/changelog/116868.yaml
@@ -0,0 +1,5 @@
+pr: 116868
+summary: Run `TransportGetComponentTemplateAction` on local node
+area: Indices APIs
+type: enhancement
+issues: []
diff --git a/docs/changelog/117214.yaml b/docs/changelog/117214.yaml
@@ -0,0 +1,5 @@
+pr: 117214
+summary: Returning ignored fields in the simulate ingest API
+area: Ingest Node
+type: enhancement
+issues: []
diff --git a/docs/changelog/118016.yaml b/docs/changelog/118016.yaml
@@ -0,0 +1,6 @@
+pr: 118016
+summary: Propagate status codes from shard failures appropriately
+area: Search
+type: enhancement
+issues:
+ - 118482
diff --git a/docs/changelog/118870.yaml b/docs/changelog/118870.yaml
@@ -0,0 +1,6 @@
+pr: 118870
+summary: Rewrite TO_UPPER/TO_LOWER comparisons
+area: ES|QL
+type: enhancement
+issues:
+ - 118304
diff --git a/docs/changelog/118958.yaml b/docs/changelog/118958.yaml
@@ -0,0 +1,5 @@
+pr: 118958
+summary: Add missing timeouts to rest-api-spec SLM APIs
+area: ILM+SLM
+type: bug
+issues: []
diff --git a/docs/changelog/118999.yaml b/docs/changelog/118999.yaml
@@ -0,0 +1,6 @@
+pr: 118999
+summary: Fix loss of context in the inference API for streaming APIs
+area: Machine Learning
+type: bug
+issues:
+ - 119000
diff --git a/docs/changelog/119134.yaml b/docs/changelog/119134.yaml
@@ -0,0 +1,6 @@
+pr: 119134
+summary: Handle `index.mapping.ignore_malformed` in downsampling
+area: Downsampling
+type: bug
+issues:
+ - 119075
diff --git a/docs/changelog/119265.yaml b/docs/changelog/119265.yaml
@@ -0,0 +1,6 @@
+pr: 119265
+summary: Fix `AbstractShapeGeometryFieldMapperTests`
+area: "ES|QL"
+type: bug
+issues:
+ - 119201
diff --git a/docs/changelog/119291.yaml b/docs/changelog/119291.yaml
@@ -0,0 +1,5 @@
+pr: 119291
+summary: Register mustache size limit setting
+area: Infra/Scripting
+type: bug
+issues: []
diff --git a/docs/changelog/119296.yaml b/docs/changelog/119296.yaml
@@ -0,0 +1,6 @@
+pr: 119296
+summary: Fix writing for LOOKUP status
+area: ES|QL
+type: bug
+issues:
+ - 119086
diff --git a/docs/reference/indices/get-component-template.asciidoc b/docs/reference/indices/get-component-template.asciidoc
@@ -67,7 +67,7 @@ Wildcard (`*`) expressions are supported.
 
 include::{docdir}/rest-api/common-parms.asciidoc[tag=flat-settings]
 
-include::{docdir}/rest-api/common-parms.asciidoc[tag=local]
+include::{docdir}/rest-api/common-parms.asciidoc[tag=local-deprecated-9.0.0]
 
 include::{docdir}/rest-api/common-parms.asciidoc[tag=master-timeout]
 

diff --git a/docs/reference/ingest/enrich.asciidoc b/docs/reference/ingest/enrich.asciidoc
@@ -98,7 +98,8 @@ and <<update-enrich-policies, update your enrich policies>>.
 [IMPORTANT]
 ====
 The enrich processor performs several operations and may impact the speed of
-your ingest pipeline.
+your ingest pipeline. We recommend <<modules-node,node roles>> co-locating
+ingest and data roles to minimize remote search operations.
 
 We strongly recommend testing and benchmarking your enrich processors
 before deploying them in production.

diff --git a/docs/reference/rest-api/common-parms.asciidoc b/docs/reference/rest-api/common-parms.asciidoc
@@ -668,6 +668,16 @@ node only. Defaults to `false`, which means information is retrieved from
 the master node.
 end::local[]
 
+tag::local-deprecated-9.0.0[]
+`local`::
+(Optional, Boolean) If `true`, the request retrieves information from the local
+node only. Defaults to `false`, which means information is retrieved from
+the master node.
++
+deprecated::[9.0.0, "The `?local` query parameter to this API has no effect, is now deprecated, and will be removed in a future version."]
+
+end::local-deprecated-9.0.0[]
+
 tag::mappings[]
 `mappings`::
 +

diff --git a/docs/reference/rest-api/usage.asciidoc b/docs/reference/rest-api/usage.asciidoc
@@ -207,16 +207,8 @@ GET /_xpack/usage
   "inference": {
     "available" : true,
     "enabled" : true,
-    "models" : [{
-        "service": "elasticsearch",
-        "task_type": "SPARSE_EMBEDDING",
-        "count": 1
-      },
-      {
-        "service": "elasticsearch",
-        "task_type": "TEXT_EMBEDDING",
-        "count": 1
-      },
+    "models" : [
+      ...
     ]
   },
   "logstash" : {
@@ -523,7 +515,10 @@ GET /_xpack/usage
     "available": true,
     "enabled": false,
     "indices_count": 0,
-    "indices_with_synthetic_source": 0
+    "indices_with_synthetic_source": 0,
+    "num_docs": 0,
+    "size_in_bytes": 0,
+    "has_custom_cutoff_date": false
   }
 }
 ------------------------------------------------------------
@@ -535,6 +530,7 @@ GET /_xpack/usage
 // TESTRESPONSE[s/"policy_stats" : \[[^\]]*\]/"policy_stats" : $body.$_path/]
 // TESTRESPONSE[s/"slm" : \{[^\}]*\},/"slm" : $body.$_path,/]
 // TESTRESPONSE[s/"health_api" : \{[^\}]*\}\s*\}/"health_api" : $body.$_path/]
+// TESTRESPONSE[s/"models" : \[[^\]]*\]/"models" : $body.$_path/]
 // TESTRESPONSE[s/"data_streams" : \{[^\}]*\},/"data_streams" : $body.$_path,/]
 // TESTRESPONSE[s/ : true/ : $body.$_path/]
 // TESTRESPONSE[s/ : false/ : $body.$_path/]
@@ -551,4 +547,5 @@ GET /_xpack/usage
 // 5. All of the numbers and strings on the right hand side of *every* field in
 //    the response are ignored. So we're really only asserting things about the
 //    the shape of this response, not the values in it.
-// 6. Ignore the contents of data streams until the failure store is tech preview.
+// 6. Ignore the contents of the `inference.models` array because the models might not yet have been initialized
+// 7. Ignore the contents of data streams until the failure store is tech preview.
diff --git a/libs/grok/src/main/java/org/elasticsearch/grok/PatternBank.java b/libs/grok/src/main/java/org/elasticsearch/grok/PatternBank.java
@@ -23,7 +23,7 @@
 
 public class PatternBank {
 
-    public static PatternBank EMPTY = new PatternBank(Map.of());
+    public static final PatternBank EMPTY = new PatternBank(Map.of());
 
     private final Map<String, String> bank;
 

diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/Constants.java b/libs/h3/src/main/java/org/elasticsearch/h3/Constants.java
@@ -29,27 +29,27 @@ final class Constants {
     /**
      * sqrt(3) / 2.0
      */
-    public static double M_SQRT3_2 = 0.8660254037844386467637231707529361834714;
+    public static final double M_SQRT3_2 = 0.8660254037844386467637231707529361834714;
     /**
      * 2.0 * PI
      */
     public static final double M_2PI = 2.0 * Math.PI;
     /**
      * The number of H3 base cells
      */
-    public static int NUM_BASE_CELLS = 122;
+    public static final int NUM_BASE_CELLS = 122;
     /**
      * The number of vertices in a hexagon
      */
-    public static int NUM_HEX_VERTS = 6;
+    public static final int NUM_HEX_VERTS = 6;
     /**
      * The number of vertices in a pentagon
      */
-    public static int NUM_PENT_VERTS = 5;
+    public static final int NUM_PENT_VERTS = 5;
     /**
      * H3 index modes
      */
-    public static int H3_CELL_MODE = 1;
+    public static final int H3_CELL_MODE = 1;
     /**
      * square root of 7
      */
@@ -64,14 +64,14 @@ final class Constants {
      * (or distance between adjacent cell center points
      * on the plane) to gnomonic unit length.
      */
-    public static double RES0_U_GNOMONIC = 0.38196601125010500003;
+    public static final double RES0_U_GNOMONIC = 0.38196601125010500003;
     /**
      * rotation angle between Class II and Class III resolution axes
      * (asin(sqrt(3.0 / 28.0)))
      */
-    public static double M_AP7_ROT_RADS = 0.333473172251832115336090755351601070065900389;
+    public static final double M_AP7_ROT_RADS = 0.333473172251832115336090755351601070065900389;
     /**
      * threshold epsilon
      */
-    public static double EPSILON = 0.0000000000000001;
+    public static final double EPSILON = 0.0000000000000001;
 }
diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/H3.java b/libs/h3/src/main/java/org/elasticsearch/h3/H3.java
@@ -33,7 +33,7 @@ public final class H3 {
     /**
      * max H3 resolution; H3 version 1 has 16 resolutions, numbered 0 through 15
      */
-    public static int MAX_H3_RES = 15;
+    public static final int MAX_H3_RES = 15;
 
     private static final long[] NORTH = new long[MAX_H3_RES + 1];
     private static final long[] SOUTH = new long[MAX_H3_RES + 1];