From f5ac68df95ccb712053355fdf54645edf6a00b19 Mon Sep 17 00:00:00 2001 From: Ioana Tagirta Date: Mon, 13 Jan 2025 22:18:47 +0100 Subject: [PATCH 01/31] ESQL: Document support for semantic_text field mapping (#120052) * Document support for semantic_text field mapping * Address review comments --- docs/reference/esql/esql-limitations.asciidoc | 4 ++-- docs/reference/esql/functions/description/match.asciidoc | 2 +- docs/reference/esql/functions/kibana/definition/match.json | 2 +- docs/reference/esql/functions/kibana/docs/match.md | 3 ++- .../xpack/esql/expression/function/fulltext/Match.java | 3 ++- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/reference/esql/esql-limitations.asciidoc b/docs/reference/esql/esql-limitations.asciidoc index 8ce8064a8161c..adfd38478ab21 100644 --- a/docs/reference/esql/esql-limitations.asciidoc +++ b/docs/reference/esql/esql-limitations.asciidoc @@ -30,11 +30,11 @@ include::processing-commands/limit.asciidoc[tag=limitation] ** You can use `to_datetime` to cast to millisecond dates to use unsupported functions * `double` (`float`, `half_float`, `scaled_float` are represented as `double`) * `ip` -* `keyword` family including `keyword`, `constant_keyword`, and `wildcard` +* `keyword` <> including `keyword`, `constant_keyword`, and `wildcard` * `int` (`short` and `byte` are represented as `int`) * `long` * `null` -* `text` +* `text` <> including `text`, `semantic_text` and `match_only_text` * experimental:[] `unsigned_long` * `version` * Spatial types diff --git a/docs/reference/esql/functions/description/match.asciidoc b/docs/reference/esql/functions/description/match.asciidoc index 931fd5eb2f94a..0724f0f108e3c 100644 --- a/docs/reference/esql/functions/description/match.asciidoc +++ b/docs/reference/esql/functions/description/match.asciidoc @@ -2,4 +2,4 @@ *Description* -Use `MATCH` to perform a <> on the specified field. Using `MATCH` is equivalent to using the `match` query in the Elasticsearch Query DSL. Match can be used on text fields, as well as other field types like boolean, dates, and numeric types. For a simplified syntax, you can use the <> `:` operator instead of `MATCH`. `MATCH` returns true if the provided query matches the row. +Use `MATCH` to perform a <> on the specified field. Using `MATCH` is equivalent to using the `match` query in the Elasticsearch Query DSL. Match can be used on fields from the text family like <> and <>, as well as other field types like keyword, boolean, dates, and numeric types. For a simplified syntax, you can use the <> `:` operator instead of `MATCH`. `MATCH` returns true if the provided query matches the row. diff --git a/docs/reference/esql/functions/kibana/definition/match.json b/docs/reference/esql/functions/kibana/definition/match.json index d61534da81a6d..eb206cb9ddf4d 100644 --- a/docs/reference/esql/functions/kibana/definition/match.json +++ b/docs/reference/esql/functions/kibana/definition/match.json @@ -2,7 +2,7 @@ "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", "type" : "eval", "name" : "match", - "description" : "Use `MATCH` to perform a <> on the specified field.\nUsing `MATCH` is equivalent to using the `match` query in the Elasticsearch Query DSL.\n\nMatch can be used on text fields, as well as other field types like boolean, dates, and numeric types.\n\nFor a simplified syntax, you can use the <> `:` operator instead of `MATCH`.\n\n`MATCH` returns true if the provided query matches the row.", + "description" : "Use `MATCH` to perform a <> on the specified field.\nUsing `MATCH` is equivalent to using the `match` query in the Elasticsearch Query DSL.\n\nMatch can be used on fields from the text family like <> and <>,\nas well as other field types like keyword, boolean, dates, and numeric types.\n\nFor a simplified syntax, you can use the <> `:` operator instead of `MATCH`.\n\n`MATCH` returns true if the provided query matches the row.", "signatures" : [ { "params" : [ diff --git a/docs/reference/esql/functions/kibana/docs/match.md b/docs/reference/esql/functions/kibana/docs/match.md index 72258a1682936..80bf84351c188 100644 --- a/docs/reference/esql/functions/kibana/docs/match.md +++ b/docs/reference/esql/functions/kibana/docs/match.md @@ -6,7 +6,8 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ Use `MATCH` to perform a <> on the specified field. Using `MATCH` is equivalent to using the `match` query in the Elasticsearch Query DSL. -Match can be used on text fields, as well as other field types like boolean, dates, and numeric types. +Match can be used on fields from the text family like <> and <>, +as well as other field types like keyword, boolean, dates, and numeric types. For a simplified syntax, you can use the <> `:` operator instead of `MATCH`. diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java index 7552b100119f0..93f23d2f7ad08 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java @@ -103,7 +103,8 @@ public class Match extends FullTextFunction implements PostOptimizationVerificat Use `MATCH` to perform a <> on the specified field. Using `MATCH` is equivalent to using the `match` query in the Elasticsearch Query DSL. - Match can be used on text fields, as well as other field types like boolean, dates, and numeric types. + Match can be used on fields from the text family like <> and <>, + as well as other field types like keyword, boolean, dates, and numeric types. For a simplified syntax, you can use the <> `:` operator instead of `MATCH`. From 5167497f50b1892bd3fc50b15c57dff643eff33a Mon Sep 17 00:00:00 2001 From: weizijun Date: Tue, 14 Jan 2025 06:54:22 +0800 Subject: [PATCH 02/31] [Inference API] fix spell words: covertToString to convertToString (#119922) --- docs/changelog/119922.yaml | 5 +++++ .../AlibabaCloudSearchEmbeddingsRequestEntity.java | 4 ++-- .../AlibabaCloudSearchSparseRequestEntity.java | 2 +- .../request/cohere/CohereEmbeddingsRequestEntity.java | 4 ++-- .../AlibabaCloudSearchEmbeddingsRequestEntityTests.java | 2 +- .../request/cohere/CohereEmbeddingsRequestEntityTests.java | 5 ++++- 6 files changed, 15 insertions(+), 7 deletions(-) create mode 100644 docs/changelog/119922.yaml diff --git a/docs/changelog/119922.yaml b/docs/changelog/119922.yaml new file mode 100644 index 0000000000000..2fc9d9529c968 --- /dev/null +++ b/docs/changelog/119922.yaml @@ -0,0 +1,5 @@ +pr: 119922 +summary: "[Inference API] fix spell words: covertToString to convertToString" +area: Machine Learning +type: enhancement +issues: [] diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestEntity.java index 1fc61d3331d20..0804cb332b15d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestEntity.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestEntity.java @@ -39,7 +39,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(); builder.field(TEXTS_FIELD, input); - String inputType = covertToString(taskSettings.getInputType()); + String inputType = convertToString(taskSettings.getInputType()); if (inputType != null) { builder.field(INPUT_TYPE_FIELD, inputType); } @@ -49,7 +49,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } // default for testing - static String covertToString(InputType inputType) { + static String convertToString(InputType inputType) { if (inputType == null) { return null; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchSparseRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchSparseRequestEntity.java index 8fae9408b860d..22676c11dc548 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchSparseRequestEntity.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchSparseRequestEntity.java @@ -34,7 +34,7 @@ public record AlibabaCloudSearchSparseRequestEntity(List input, AlibabaC public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(TEXTS_FIELD, input); - String inputType = AlibabaCloudSearchEmbeddingsRequestEntity.covertToString(taskSettings.getInputType()); + String inputType = AlibabaCloudSearchEmbeddingsRequestEntity.convertToString(taskSettings.getInputType()); if (inputType != null) { builder.field(INPUT_TYPE_FIELD, inputType); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequestEntity.java index 63cc5c3cb7261..a972cbbac959d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequestEntity.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequestEntity.java @@ -51,7 +51,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } if (taskSettings.getInputType() != null) { - builder.field(INPUT_TYPE_FIELD, covertToString(taskSettings.getInputType())); + builder.field(INPUT_TYPE_FIELD, convertToString(taskSettings.getInputType())); } if (embeddingType != null) { @@ -67,7 +67,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } // default for testing - static String covertToString(InputType inputType) { + static String convertToString(InputType inputType) { return switch (inputType) { case INGEST -> SEARCH_DOCUMENT; case SEARCH -> SEARCH_QUERY; diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestEntityTests.java index 6aaab219c331d..39eeef3a95520 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestEntityTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestEntityTests.java @@ -50,7 +50,7 @@ public void testXContent_WritesNoOptionalFields_WhenTheyAreNotDefined() throws I public void testConvertToString_ThrowsAssertionFailure_WhenInputTypeIsUnspecified() { var thrownException = expectThrows( AssertionError.class, - () -> AlibabaCloudSearchEmbeddingsRequestEntity.covertToString(InputType.UNSPECIFIED) + () -> AlibabaCloudSearchEmbeddingsRequestEntity.convertToString(InputType.UNSPECIFIED) ); MatcherAssert.assertThat(thrownException.getMessage(), is("received invalid input type value [unspecified]")); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequestEntityTests.java index 0690bf56893ca..8c8aeba4a0a07 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequestEntityTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequestEntityTests.java @@ -84,7 +84,10 @@ public void testXContent_WritesNoOptionalFields_WhenTheyAreNotDefined() throws I } public void testConvertToString_ThrowsAssertionFailure_WhenInputTypeIsUnspecified() { - var thrownException = expectThrows(AssertionError.class, () -> CohereEmbeddingsRequestEntity.covertToString(InputType.UNSPECIFIED)); + var thrownException = expectThrows( + AssertionError.class, + () -> CohereEmbeddingsRequestEntity.convertToString(InputType.UNSPECIFIED) + ); MatcherAssert.assertThat(thrownException.getMessage(), is("received invalid input type value [unspecified]")); } } From 455fde2849cd2bd6c9a38bc21383417194d64205 Mon Sep 17 00:00:00 2001 From: Dianna Hohensee Date: Mon, 13 Jan 2025 19:58:55 -0500 Subject: [PATCH 03/31] Add documentation around desired balance (#119902) Relates ES-10341 --- .../routing/allocation/AllocationService.java | 9 ++---- .../allocation/allocator/DesiredBalance.java | 21 +++++++++++- .../allocator/DesiredBalanceComputer.java | 13 +++++--- .../allocator/DesiredBalanceReconciler.java | 15 +++++++++ .../DesiredBalanceShardsAllocator.java | 32 +++++++++++++++---- .../allocator/PendingListenersQueue.java | 4 +++ .../allocation/allocator/ShardAssignment.java | 14 +++++++- .../allocator/DesiredBalanceTests.java | 10 +++--- 8 files changed, 94 insertions(+), 24 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java index 5d1e6741c5e22..2f2fd4ef453f6 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java @@ -158,9 +158,7 @@ public ClusterState applyStartedShards(ClusterState clusterState, List ass * The placeholder value for {@link DesiredBalance} when the node stands down as master. */ public static final DesiredBalance NOT_MASTER = new DesiredBalance(-2, Map.of()); + /** * The starting value for {@link DesiredBalance} when the node becomes the master. */ @@ -57,6 +65,10 @@ public static boolean hasChanges(DesiredBalance a, DesiredBalance b) { return Objects.equals(a.assignments, b.assignments) == false; } + /** + * Returns the sum of shard movements needed to reach the new desired balance. Doesn't count new shard copies as a move, nor removal or + * unassignment of a shard copy. + */ public static int shardMovements(DesiredBalance old, DesiredBalance updated) { var intersection = Sets.intersection(old.assignments().keySet(), updated.assignments().keySet()); int movements = 0; @@ -70,8 +82,15 @@ public static int shardMovements(DesiredBalance old, DesiredBalance updated) { return movements; } + /** + * Returns the number of shard movements needed to reach the new shard assignment. Doesn't count new shard copies as a move, nor removal + * or unassignment of a shard copy. + */ private static int shardMovements(ShardAssignment old, ShardAssignment updated) { - var movements = Math.min(0, old.assigned() - updated.assigned());// compensate newly started shards + // A shard move should retain the same number of assigned nodes, just swap out one node for another. We will compensate for newly + // started shards -- adding a shard copy is not a move -- by initializing the count with a negative value so that incrementing later + // for a new node zeros out. + var movements = Math.min(0, old.assigned() - updated.assigned()); for (String nodeId : updated.nodeIds()) { if (old.nodeIds().contains(nodeId) == false) { movements++; diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputer.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputer.java index 3b22221ea7db4..03630c284fa30 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputer.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputer.java @@ -415,11 +415,14 @@ boolean hasEnoughIterations(int currentIteration) { } private static Map collectShardAssignments(RoutingNodes routingNodes) { - final var entries = routingNodes.getAssignedShards().entrySet(); - assert entries.stream().flatMap(t -> t.getValue().stream()).allMatch(ShardRouting::started) : routingNodes; - final Map res = Maps.newHashMapWithExpectedSize(entries.size()); - for (var shardAndAssignments : entries) { - res.put(shardAndAssignments.getKey(), ShardAssignment.ofAssignedShards(shardAndAssignments.getValue())); + final var allAssignedShards = routingNodes.getAssignedShards().entrySet(); + assert allAssignedShards.stream().flatMap(t -> t.getValue().stream()).allMatch(ShardRouting::started) : routingNodes; + final Map res = Maps.newHashMapWithExpectedSize(allAssignedShards.size()); + for (var shardIdAndShardRoutings : allAssignedShards) { + res.put( + shardIdAndShardRoutings.getKey(), + ShardAssignment.createFromAssignedShardRoutingsList(shardIdAndShardRoutings.getValue()) + ); } return res; } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java index 83b370c1a7928..909a7a7a99a61 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java @@ -54,6 +54,10 @@ public class DesiredBalanceReconciler { private static final Logger logger = LogManager.getLogger(DesiredBalanceReconciler.class); + /** + * The minimum interval that log messages will be written if the number of undesired shard allocations reaches the percentage of total + * shards set by {@link #UNDESIRED_ALLOCATIONS_LOG_THRESHOLD_SETTING}. + */ public static final Setting UNDESIRED_ALLOCATIONS_LOG_INTERVAL_SETTING = Setting.timeSetting( "cluster.routing.allocation.desired_balance.undesired_allocations.log_interval", TimeValue.timeValueHours(1), @@ -62,6 +66,10 @@ public class DesiredBalanceReconciler { Setting.Property.NodeScope ); + /** + * Warning log messages may be periodically written if the number of shards that are on undesired nodes reaches this percentage setting. + * Works together with {@link #UNDESIRED_ALLOCATIONS_LOG_INTERVAL_SETTING} to log on a periodic basis. + */ public static final Setting UNDESIRED_ALLOCATIONS_LOG_THRESHOLD_SETTING = Setting.doubleSetting( "cluster.routing.allocation.desired_balance.undesired_allocations.threshold", 0.1, @@ -96,6 +104,13 @@ public DesiredBalanceReconciler( this.nodeAllocationStatsAndWeightsCalculator = nodeAllocationStatsAndWeightsCalculator; } + /** + * Applies a desired shard allocation to the routing table by initializing and relocating shards in the cluster state. + * + * @param desiredBalance The new desired cluster shard allocation + * @param allocation Cluster state information with which to make decisions, contains routing table metadata that will be modified to + * reach the given desired balance. + */ public void reconcile(DesiredBalance desiredBalance, RoutingAllocation allocation) { var nodeIds = allocation.routingNodes().getAllNodeIds(); allocationOrdering.retainNodes(nodeIds); diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocator.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocator.java index 5be26f0b3e8c7..2c73a27ad3418 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocator.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocator.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.ClusterStateTaskListener; import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata; import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.AllocationService.RerouteStrategy; import org.elasticsearch.cluster.routing.allocation.NodeAllocationStatsAndWeightsCalculator; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; @@ -56,11 +57,30 @@ public class DesiredBalanceShardsAllocator implements ShardsAllocator { private final ShardsAllocator delegateAllocator; private final ThreadPool threadPool; + /** + * This is a callback to run {@link AllocationService#executeWithRoutingAllocation(ClusterState, String, RerouteStrategy)}, which + * produces a new ClusterState with the changes made by {@link DesiredBalanceReconciler#reconcile}. The {@link RerouteStrategy} provided + * to the callback calls into {@link #desiredBalanceReconciler} for the changes. The {@link #masterServiceTaskQueue} will publish the + * new cluster state after the cluster state is constructed by the {@link ReconcileDesiredBalanceExecutor}. + */ private final DesiredBalanceReconcilerAction reconciler; private final DesiredBalanceComputer desiredBalanceComputer; + /** + * Reconciliation ({@link DesiredBalanceReconciler#reconcile(DesiredBalance, RoutingAllocation)}) takes the {@link DesiredBalance} + * output of {@link DesiredBalanceComputer#compute} and identifies how shards need to be added, moved or removed to go from the current + * cluster shard allocation to the new desired allocation. + */ private final DesiredBalanceReconciler desiredBalanceReconciler; private final ContinuousComputation desiredBalanceComputation; - private final PendingListenersQueue queue; + /** + * Saves and runs listeners after DesiredBalance computations complete. + */ + private final PendingListenersQueue pendingListenersQueue; + /** + * Each reroute request gets assigned a monotonically increasing sequence number. Many reroute requests may arrive before the balancer + * asynchronously runs a computation. The balancer will use the latest request and save this sequence number to track back to the + * request. + */ private final AtomicLong indexGenerator = new AtomicLong(-1); private final ConcurrentLinkedQueue> pendingDesiredBalanceMoves = new ConcurrentLinkedQueue<>(); private final MasterServiceTaskQueue masterServiceTaskQueue; @@ -199,7 +219,7 @@ public String toString() { return "DesiredBalanceShardsAllocator#allocate"; } }; - this.queue = new PendingListenersQueue(); + this.pendingListenersQueue = new PendingListenersQueue(); this.masterServiceTaskQueue = clusterService.createTaskQueue( "reconcile-desired-balance", Priority.URGENT, @@ -235,7 +255,7 @@ public void allocate(RoutingAllocation allocation, ActionListener listener var index = indexGenerator.incrementAndGet(); logger.debug("Executing allocate for [{}]", index); - queue.add(index, listener); + pendingListenersQueue.add(index, listener); // This can only run on master, so unset not-master if exists if (currentDesiredBalanceRef.compareAndSet(DesiredBalance.NOT_MASTER, DesiredBalance.BECOME_MASTER_INITIAL)) { logger.debug("initialized desired balance for becoming master"); @@ -378,7 +398,7 @@ public DesiredBalanceStats getStats() { private void onNoLongerMaster() { if (indexGenerator.getAndSet(-1) != -1) { currentDesiredBalanceRef.set(DesiredBalance.NOT_MASTER); - queue.completeAllAsNotMaster(); + pendingListenersQueue.completeAllAsNotMaster(); pendingDesiredBalanceMoves.clear(); desiredBalanceReconciler.clear(); desiredBalanceMetrics.zeroAllMetrics(); @@ -428,7 +448,7 @@ private ClusterState applyBalance( batchExecutionContext.initialState(), createReconcileAllocationAction(latest.getTask().desiredBalance) ); - latest.success(() -> queue.complete(latest.getTask().desiredBalance.lastConvergedIndex())); + latest.success(() -> pendingListenersQueue.complete(latest.getTask().desiredBalance.lastConvergedIndex())); return newState; } } @@ -447,7 +467,7 @@ private static void discardSupersededTasks( // only for tests - in production, this happens after reconciliation protected final void completeToLastConvergedIndex() { - queue.complete(currentDesiredBalanceRef.get().lastConvergedIndex()); + pendingListenersQueue.complete(currentDesiredBalanceRef.get().lastConvergedIndex()); } private void recordTime(CounterMetric metric, Runnable action) { diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/PendingListenersQueue.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/PendingListenersQueue.java index e1b58cf79ac09..5b14277f2c651 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/PendingListenersQueue.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/PendingListenersQueue.java @@ -24,6 +24,10 @@ import static org.elasticsearch.cluster.service.ClusterApplierService.CLUSTER_UPDATE_THREAD_NAME; import static org.elasticsearch.cluster.service.MasterService.MASTER_UPDATE_THREAD_NAME; +/** + * Registers listeners with an `index` number ({@link #add(long, ActionListener)}) and then completes them whenever the latest index number + * is greater or equal to a listener's index value ({@link #complete(long)}). + */ public class PendingListenersQueue { private static final Logger logger = LogManager.getLogger(PendingListenersQueue.class); diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardAssignment.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardAssignment.java index 4fb9137cb4544..2bd1b9bb2bb64 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardAssignment.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardAssignment.java @@ -17,6 +17,14 @@ import static java.util.Collections.unmodifiableSet; +/** + * Simple shard assignment summary of shard copies for a particular index shard. + * + * @param nodeIds The node IDs of nodes holding a shard copy. + * @param total The total number of shard copies. + * @param unassigned The number of unassigned shard copies. + * @param ignored The number of ignored shard copies. + */ public record ShardAssignment(Set nodeIds, int total, int unassigned, int ignored) { public ShardAssignment { @@ -28,9 +36,13 @@ public int assigned() { return nodeIds.size(); } - public static ShardAssignment ofAssignedShards(List routings) { + /** + * Helper method to instantiate a new ShardAssignment from a given list of ShardRouting instances. Assumes all shards are assigned. + */ + public static ShardAssignment createFromAssignedShardRoutingsList(List routings) { var nodeIds = new LinkedHashSet(); for (ShardRouting routing : routings) { + assert routing.unassignedInfo() == null : "Expected assigned shard copies only, unassigned info: " + routing.unassignedInfo(); nodeIds.add(routing.currentNodeId()); } return new ShardAssignment(unmodifiableSet(nodeIds), routings.size(), 0, 0); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceTests.java index 760900817780a..2c15addfe217b 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceTests.java @@ -46,19 +46,19 @@ public void testShardMovements() { ); assertThat( - "1 shard movements when existing shard is moved and new shard copy is unassigned", + "1 shard movements when an existing shard copy is moved and new shard copy is unassigned", shardMovements(new ShardAssignment(Set.of("a", "b"), 2, 0, 0), new ShardAssignment(Set.of("a", "c"), 3, 1, 0)), equalTo(1) ); assertThat( - "1 shard movement", + "1 shard movement when an existing shard copy is moved", shardMovements(new ShardAssignment(Set.of("a", "b"), 2, 0, 0), new ShardAssignment(Set.of("a", "c"), 2, 0, 0)), equalTo(1) ); assertThat( - "2 shard movement", + "2 shard movements when both shard copies are move to new nodes", shardMovements(new ShardAssignment(Set.of("a", "b"), 2, 0, 0), new ShardAssignment(Set.of("c", "d"), 2, 0, 0)), equalTo(2) ); @@ -77,10 +77,10 @@ public void testShardMovements() { } private static int shardMovements(ShardAssignment old, ShardAssignment updated) { - return DesiredBalance.shardMovements(of(old), of(updated)); + return DesiredBalance.shardMovements(createDesiredBalanceWith(old), createDesiredBalanceWith(updated)); } - private static DesiredBalance of(ShardAssignment assignment) { + private static DesiredBalance createDesiredBalanceWith(ShardAssignment assignment) { return new DesiredBalance(1, Map.of(new ShardId("index", "_na_", 0), assignment)); } } From 4605ecf44de75ff7a5b6ec060b9540ab6e3394ee Mon Sep 17 00:00:00 2001 From: Niels Bauman <33722607+nielsbauman@users.noreply.github.com> Date: Tue, 14 Jan 2025 12:39:41 +1000 Subject: [PATCH 04/31] Remove trappy timeouts from `GetMappingsRequest` (#120035) Relates #107984 --- .../datastreams/DataStreamIT.java | 8 ++-- .../index/mapper/size/SizeMappingIT.java | 2 +- .../action/IndicesRequestIT.java | 2 +- .../admin/indices/create/CreateIndexIT.java | 6 +-- .../indices/create/CreateSystemIndicesIT.java | 5 ++- .../action/bulk/BulkIntegrationIT.java | 2 +- .../cluster/SimpleClusterStateIT.java | 7 ++- .../gateway/MetadataNodesIT.java | 4 +- .../elasticsearch/index/HiddenIndexIT.java | 2 +- .../index/mapper/DynamicMappingIT.java | 26 ++++++++--- .../mapper/MultiFieldsIntegrationIT.java | 10 ++--- .../indices/IndicesOptionsIntegrationIT.java | 26 +++++------ .../SystemIndexMappingUpdateServiceIT.java | 5 ++- .../mapping/SimpleGetFieldMappingsIT.java | 2 +- .../indices/mapping/SimpleGetMappingsIT.java | 12 ++--- .../mapping/UpdateMappingIntegrationIT.java | 11 +++-- .../mapping/get/GetMappingsRequest.java | 5 ++- .../get/GetMappingsRequestBuilder.java | 5 ++- .../master/info/ClusterInfoRequest.java | 12 +++++ .../client/internal/IndicesAdminClient.java | 4 +- .../admin/indices/RestGetMappingAction.java | 5 +-- .../mapper/FieldFilterMapperPluginTests.java | 8 ++-- .../mapper/NonDynamicFieldMapperTestCase.java | 4 +- .../xpack/ccr/CcrRepositoryIT.java | 2 +- .../xpack/ccr/IndexFollowingIT.java | 23 ++++++++-- .../sourceonly/SourceOnlySnapshotIT.java | 2 +- .../downsample/TransportDownsampleAction.java | 2 +- .../DownsampleActionSingleNodeTests.java | 7 +-- .../action/CreateIndexFromSourceActionIT.java | 5 ++- ...indexDatastreamIndexTransportActionIT.java | 6 ++- .../ml/integration/JobResultsProviderIT.java | 4 +- ...ransportStartDataFrameAnalyticsAction.java | 5 ++- .../xpack/ml/dataframe/DestinationIndex.java | 2 + .../xpack/ml/dataframe/MappingsMerger.java | 4 +- .../job/persistence/JobResultsProvider.java | 6 ++- .../action/TransportPutRollupJobAction.java | 7 +-- .../action/PutJobStateMachineTests.java | 45 ++++++++++++++++--- .../DocumentAndFieldLevelSecurityTests.java | 8 ++-- .../integration/KibanaUserRoleIntegTests.java | 2 +- .../HistoryTemplateHttpMappingsTests.java | 3 +- .../HistoryTemplateTimeMappingsTests.java | 2 +- .../integration/HistoryIntegrationTests.java | 6 +-- 42 files changed, 208 insertions(+), 106 deletions(-) diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java index 4a3dfac36d4ec..2739eb51376ea 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java @@ -596,7 +596,7 @@ public void testResolvabilityOfDataStreamsInAPIs() throws Exception { verifyResolvability(dataStreamName, indicesAdmin().prepareGetFieldMappings(dataStreamName), false); verifyResolvability(dataStreamName, indicesAdmin().preparePutMapping(dataStreamName).setSource(""" {"_doc":{"properties": {"my_field":{"type":"keyword"}}}}""", XContentType.JSON), false); - verifyResolvability(dataStreamName, indicesAdmin().prepareGetMappings(dataStreamName), false); + verifyResolvability(dataStreamName, indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, dataStreamName), false); verifyResolvability( dataStreamName, indicesAdmin().prepareUpdateSettings(dataStreamName).setSettings(Settings.builder().put("index.number_of_replicas", 0)), @@ -643,7 +643,7 @@ public void testResolvabilityOfDataStreamsInAPIs() throws Exception { verifyResolvability(wildcardExpression, indicesAdmin().prepareGetFieldMappings(wildcardExpression), false); verifyResolvability(wildcardExpression, indicesAdmin().preparePutMapping(wildcardExpression).setSource(""" {"_doc":{"properties": {"my_field":{"type":"keyword"}}}}""", XContentType.JSON), false); - verifyResolvability(wildcardExpression, indicesAdmin().prepareGetMappings(wildcardExpression), false); + verifyResolvability(wildcardExpression, indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, wildcardExpression), false); verifyResolvability(wildcardExpression, indicesAdmin().prepareGetSettings(wildcardExpression), false); verifyResolvability( wildcardExpression, @@ -1180,7 +1180,7 @@ public void testUpdateMappingViaDataStream() throws Exception { DataStreamTimestampFieldMapper.NAME, Map.of("enabled", true) ); - GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings("logs-foobar").get(); + GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "logs-foobar").get(); assertThat(getMappingsResponse.getMappings().size(), equalTo(2)); assertThat(getMappingsResponse.getMappings().get(backingIndex1).getSourceAsMap(), equalTo(expectedMapping)); assertThat(getMappingsResponse.getMappings().get(backingIndex2).getSourceAsMap(), equalTo(expectedMapping)); @@ -1195,7 +1195,7 @@ public void testUpdateMappingViaDataStream() throws Exception { .setSource("{\"properties\":{\"my_field\":{\"type\":\"keyword\"}}}", XContentType.JSON) .get(); // The mappings of all backing indices should be updated: - getMappingsResponse = indicesAdmin().prepareGetMappings("logs-foobar").get(); + getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "logs-foobar").get(); assertThat(getMappingsResponse.getMappings().size(), equalTo(2)); assertThat(getMappingsResponse.getMappings().get(backingIndex1).getSourceAsMap(), equalTo(expectedMapping)); assertThat(getMappingsResponse.getMappings().get(backingIndex2).getSourceAsMap(), equalTo(expectedMapping)); diff --git a/plugins/mapper-size/src/internalClusterTest/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java b/plugins/mapper-size/src/internalClusterTest/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java index c2251910c3122..435849821691e 100644 --- a/plugins/mapper-size/src/internalClusterTest/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java +++ b/plugins/mapper-size/src/internalClusterTest/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java @@ -91,7 +91,7 @@ private void assertSizeMappingEnabled(String index, boolean enabled) throws IOEx "Expected size field mapping to be " + (enabled ? "enabled" : "disabled") + " for %s", index ); - GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(index).get(); + GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, index).get(); Map mappingSource = getMappingsResponse.getMappings().get(index).getSourceAsMap(); assertThat(errMsg, mappingSource, hasKey("_size")); String sizeAsString = mappingSource.get("_size").toString(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java index 420f6427a55e1..68e65b16aa3a2 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java @@ -519,7 +519,7 @@ public void testDeleteIndex() { public void testGetMappings() { interceptTransportActions(GetMappingsAction.NAME); - GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices(randomIndicesOrAliases()); + GetMappingsRequest getMappingsRequest = new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(randomIndicesOrAliases()); internalCluster().coordOnlyNodeClient().admin().indices().getMappings(getMappingsRequest).actionGet(); clearInterceptedActions(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java index 4c25b3262d559..b14bf38f3cbcc 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java @@ -120,7 +120,7 @@ public void testNonNestedMappings() throws Exception { ) ); - GetMappingsResponse response = indicesAdmin().prepareGetMappings("test").get(); + GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get(); MappingMetadata mappings = response.mappings().get("test"); assertNotNull(mappings); @@ -130,7 +130,7 @@ public void testNonNestedMappings() throws Exception { public void testEmptyNestedMappings() throws Exception { assertAcked(prepareCreate("test").setMapping(XContentFactory.jsonBuilder().startObject().endObject())); - GetMappingsResponse response = indicesAdmin().prepareGetMappings("test").get(); + GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get(); MappingMetadata mappings = response.mappings().get("test"); assertNotNull(mappings); @@ -150,7 +150,7 @@ public void testEmptyMappings() throws Exception { prepareCreate("test").setMapping(XContentFactory.jsonBuilder().startObject().startObject("_doc").endObject().endObject()) ); - GetMappingsResponse response = indicesAdmin().prepareGetMappings("test").get(); + GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get(); MappingMetadata mappings = response.mappings().get("test"); assertNotNull(mappings); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java index 867ca89f9e7f3..b9dadf86c3345 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java @@ -357,8 +357,9 @@ private void assertHasAliases(Set aliasNames, String name, String primar * Fetch the mappings and settings for {@link TestSystemIndexDescriptor#INDEX_NAME} and verify that they match the expected values. */ private void assertMappingsAndSettings(String expectedMappings, String concreteIndex) { - final GetMappingsResponse getMappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest().indices(INDEX_NAME)) - .actionGet(); + final GetMappingsResponse getMappingsResponse = indicesAdmin().getMappings( + new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(INDEX_NAME) + ).actionGet(); final Map mappings = getMappingsResponse.getMappings(); assertThat( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java index e45555b1dec19..2cd319d148321 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java @@ -55,7 +55,7 @@ public void testBulkIndexCreatesMapping() throws Exception { bulkBuilder.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, XContentType.JSON); bulkBuilder.get(); assertBusy(() -> { - GetMappingsResponse mappingsResponse = indicesAdmin().prepareGetMappings().get(); + GetMappingsResponse mappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT).get(); assertTrue(mappingsResponse.getMappings().containsKey("logstash-2014.03.30")); }); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleClusterStateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleClusterStateIT.java index 7777dd778a6c1..9eed1f757b5b1 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleClusterStateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleClusterStateIT.java @@ -258,11 +258,14 @@ public void testLargeClusterStatePublishing() throws Exception { .setTimeout(TimeValue.timeValueMinutes(1)) ); ensureGreen(); // wait for green state, so its both green, and there are no more pending events - MappingMetadata masterMappingMetadata = indicesAdmin().prepareGetMappings("test").get().getMappings().get("test"); + MappingMetadata masterMappingMetadata = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test") + .get() + .getMappings() + .get("test"); for (Client client : clients()) { MappingMetadata mappingMetadata = client.admin() .indices() - .prepareGetMappings("test") + .prepareGetMappings(TEST_REQUEST_TIMEOUT, "test") .setLocal(true) .get() .getMappings() diff --git a/server/src/internalClusterTest/java/org/elasticsearch/gateway/MetadataNodesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/gateway/MetadataNodesIT.java index d6ccdf3dc0399..256566045c59a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/gateway/MetadataNodesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/gateway/MetadataNodesIT.java @@ -115,7 +115,7 @@ public void testMetaWrittenWhenIndexIsClosedAndMetaUpdated() throws Exception { ) .get(); - GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(index).get(); + GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, index).get(); assertNotNull( ((Map) (getMappingsResponse.getMappings().get(index).getSourceAsMap().get("properties"))).get("integer_field") ); @@ -146,7 +146,7 @@ public void testMetaWrittenWhenIndexIsClosedAndMetaUpdated() throws Exception { ) .get(); - getMappingsResponse = indicesAdmin().prepareGetMappings(index).get(); + getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, index).get(); assertNotNull( ((Map) (getMappingsResponse.getMappings().get(index).getSourceAsMap().get("properties"))).get("float_field") ); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/HiddenIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/HiddenIndexIT.java index 802ba04375c48..24af560f608d3 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/HiddenIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/HiddenIndexIT.java @@ -97,7 +97,7 @@ public void testGlobalTemplatesDoNotApply() { assertAcked(indicesAdmin().prepareCreate("a_hidden_index").setSettings(Settings.builder().put("index.hidden", true).build())); - GetMappingsResponse mappingsResponse = indicesAdmin().prepareGetMappings("a_hidden_index").get(); + GetMappingsResponse mappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "a_hidden_index").get(); assertThat(mappingsResponse.mappings().size(), is(1)); MappingMetadata mappingMetadata = mappingsResponse.mappings().get("a_hidden_index"); assertNotNull(mappingMetadata); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java index a37fb25052ac5..06561bc6d4c97 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java @@ -96,7 +96,11 @@ public void testSimpleDynamicMappingsSuccessful() { client().prepareIndex("index").setId("1").setSource("a.x", 1).get(); client().prepareIndex("index").setId("2").setSource("a.y", 2).get(); - Map mappings = indicesAdmin().prepareGetMappings("index").get().mappings().get("index").sourceAsMap(); + Map mappings = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "index") + .get() + .mappings() + .get("index") + .sourceAsMap(); assertTrue(new WriteField("properties.a", () -> mappings).exists()); assertTrue(new WriteField("properties.a.properties.x", () -> mappings).exists()); } @@ -183,7 +187,7 @@ private Map indexConcurrently(int numberOfFieldsToCreate, Settin for (int i = 0; i < numberOfFieldsToCreate; ++i) { assertTrue(client().prepareGet("index", Integer.toString(i)).get().isExists()); } - GetMappingsResponse mappings = indicesAdmin().prepareGetMappings("index").get(); + GetMappingsResponse mappings = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "index").get(); MappingMetadata indexMappings = mappings.getMappings().get("index"); assertNotNull(indexMappings); Map typeMappingsMap = indexMappings.getSourceAsMap(); @@ -214,7 +218,11 @@ public void testConcurrentDynamicMappingsWithConflictingType() throws Throwable for (int i = 0; i < numberOfDocsToCreate; ++i) { assertTrue(client().prepareGet("index", Integer.toString(i)).get().isExists()); } - Map index = indicesAdmin().prepareGetMappings("index").get().getMappings().get("index").getSourceAsMap(); + Map index = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "index") + .get() + .getMappings() + .get("index") + .getSourceAsMap(); for (int i = 0, j = 1; i < numberOfDocsToCreate; i++, j++) { assertThat(new WriteField("properties.field" + i + ".type", () -> index).get(null), is(oneOf("long", "float"))); assertThat(new WriteField("properties.field" + j + ".type", () -> index).get(null), is(oneOf("long", "float"))); @@ -806,7 +814,11 @@ public void testSubobjectsFalseAtRoot() throws Exception { assertEquals(RestStatus.CREATED, indexResponse.status()); assertBusy(() -> { - Map mappings = indicesAdmin().prepareGetMappings("test").get().mappings().get("test").sourceAsMap(); + Map mappings = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test") + .get() + .mappings() + .get("test") + .sourceAsMap(); @SuppressWarnings("unchecked") Map properties = (Map) mappings.get("properties"); assertEquals(4, properties.size()); @@ -851,7 +863,11 @@ public void testSubobjectsFalse() throws Exception { assertEquals(RestStatus.CREATED, indexResponse.status()); assertBusy(() -> { - Map mappings = indicesAdmin().prepareGetMappings("test").get().mappings().get("test").sourceAsMap(); + Map mappings = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test") + .get() + .mappings() + .get("test") + .sourceAsMap(); Map properties = (Map) mappings.get("properties"); Map foo = (Map) properties.get("foo"); properties = (Map) foo.get("properties"); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/MultiFieldsIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/MultiFieldsIntegrationIT.java index 4a7de4b0ebc23..1a51fc12fed8e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/MultiFieldsIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/MultiFieldsIntegrationIT.java @@ -37,7 +37,7 @@ public class MultiFieldsIntegrationIT extends ESIntegTestCase { public void testMultiFields() throws Exception { assertAcked(indicesAdmin().prepareCreate("my-index").setMapping(createTypeSource())); - GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings("my-index").get(); + GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "my-index").get(); MappingMetadata mappingMetadata = getMappingsResponse.mappings().get("my-index"); assertThat(mappingMetadata, not(nullValue())); Map mappingSource = mappingMetadata.sourceAsMap(); @@ -53,7 +53,7 @@ public void testMultiFields() throws Exception { assertAcked(indicesAdmin().preparePutMapping("my-index").setSource(createPutMappingSource())); - getMappingsResponse = indicesAdmin().prepareGetMappings("my-index").get(); + getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "my-index").get(); mappingMetadata = getMappingsResponse.mappings().get("my-index"); assertThat(mappingMetadata, not(nullValue())); mappingSource = mappingMetadata.sourceAsMap(); @@ -74,7 +74,7 @@ public void testMultiFields() throws Exception { public void testGeoPointMultiField() throws Exception { assertAcked(indicesAdmin().prepareCreate("my-index").setMapping(createMappingSource("geo_point"))); - GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings("my-index").get(); + GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "my-index").get(); MappingMetadata mappingMetadata = getMappingsResponse.mappings().get("my-index"); assertThat(mappingMetadata, not(nullValue())); Map mappingSource = mappingMetadata.sourceAsMap(); @@ -102,7 +102,7 @@ public void testGeoPointMultiField() throws Exception { public void testCompletionMultiField() throws Exception { assertAcked(indicesAdmin().prepareCreate("my-index").setMapping(createMappingSource("completion"))); - GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings("my-index").get(); + GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "my-index").get(); MappingMetadata mappingMetadata = getMappingsResponse.mappings().get("my-index"); assertThat(mappingMetadata, not(nullValue())); Map mappingSource = mappingMetadata.sourceAsMap(); @@ -123,7 +123,7 @@ public void testCompletionMultiField() throws Exception { public void testIpMultiField() throws Exception { assertAcked(indicesAdmin().prepareCreate("my-index").setMapping(createMappingSource("ip"))); - GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings("my-index").get(); + GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "my-index").get(); MappingMetadata mappingMetadata = getMappingsResponse.mappings().get("my-index"); assertThat(mappingMetadata, not(nullValue())); Map mappingSource = mappingMetadata.sourceAsMap(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java index 652f4e02ffbce..c53cf3b56f65a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java @@ -537,26 +537,26 @@ public void testPutMapping() throws Exception { } verify(indicesAdmin().preparePutMapping("foo").setSource("field", "type=text"), false); - assertThat(indicesAdmin().prepareGetMappings("foo").get().mappings().get("foo"), notNullValue()); + assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "foo").get().mappings().get("foo"), notNullValue()); verify(indicesAdmin().preparePutMapping("b*").setSource("field", "type=text"), false); - assertThat(indicesAdmin().prepareGetMappings("bar").get().mappings().get("bar"), notNullValue()); - assertThat(indicesAdmin().prepareGetMappings("barbaz").get().mappings().get("barbaz"), notNullValue()); + assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "bar").get().mappings().get("bar"), notNullValue()); + assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "barbaz").get().mappings().get("barbaz"), notNullValue()); verify(indicesAdmin().preparePutMapping("_all").setSource("field", "type=text"), false); - assertThat(indicesAdmin().prepareGetMappings("foo").get().mappings().get("foo"), notNullValue()); - assertThat(indicesAdmin().prepareGetMappings("foobar").get().mappings().get("foobar"), notNullValue()); - assertThat(indicesAdmin().prepareGetMappings("bar").get().mappings().get("bar"), notNullValue()); - assertThat(indicesAdmin().prepareGetMappings("barbaz").get().mappings().get("barbaz"), notNullValue()); + assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "foo").get().mappings().get("foo"), notNullValue()); + assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "foobar").get().mappings().get("foobar"), notNullValue()); + assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "bar").get().mappings().get("bar"), notNullValue()); + assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "barbaz").get().mappings().get("barbaz"), notNullValue()); verify(indicesAdmin().preparePutMapping().setSource("field", "type=text"), false); - assertThat(indicesAdmin().prepareGetMappings("foo").get().mappings().get("foo"), notNullValue()); - assertThat(indicesAdmin().prepareGetMappings("foobar").get().mappings().get("foobar"), notNullValue()); - assertThat(indicesAdmin().prepareGetMappings("bar").get().mappings().get("bar"), notNullValue()); - assertThat(indicesAdmin().prepareGetMappings("barbaz").get().mappings().get("barbaz"), notNullValue()); + assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "foo").get().mappings().get("foo"), notNullValue()); + assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "foobar").get().mappings().get("foobar"), notNullValue()); + assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "bar").get().mappings().get("bar"), notNullValue()); + assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "barbaz").get().mappings().get("barbaz"), notNullValue()); verify(indicesAdmin().preparePutMapping("c*").setSource("field", "type=text"), true); assertAcked(indicesAdmin().prepareClose("barbaz").get()); verify(indicesAdmin().preparePutMapping("barbaz").setSource("field", "type=text"), false); - assertThat(indicesAdmin().prepareGetMappings("barbaz").get().mappings().get("barbaz"), notNullValue()); + assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "barbaz").get().mappings().get("barbaz"), notNullValue()); } public static final class TestPlugin extends Plugin { @@ -664,7 +664,7 @@ static GetFieldMappingsRequestBuilder getFieldMapping(String... indices) { } static GetMappingsRequestBuilder getMapping(String... indices) { - return indicesAdmin().prepareGetMappings(indices); + return indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, indices); } static GetSettingsRequestBuilder getSettings(String... indices) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceIT.java index de565605ff58a..7264585337fc7 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceIT.java @@ -123,8 +123,9 @@ private void triggerClusterStateUpdates() { * Fetch the mappings and settings for {@link TestSystemIndexDescriptor#INDEX_NAME} and verify that they match the expected values. */ private void assertMappingsAndSettings(String expectedMappings) { - final GetMappingsResponse getMappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest().indices(INDEX_NAME)) - .actionGet(); + final GetMappingsResponse getMappingsResponse = indicesAdmin().getMappings( + new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(INDEX_NAME) + ).actionGet(); final Map mappings = getMappingsResponse.getMappings(); assertThat( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsIT.java index 71fcb25c2e0b2..e3092bda185fe 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsIT.java @@ -198,7 +198,7 @@ public void testGetFieldMappingsWithBlocks() throws Exception { try { enableIndexBlock("test", SETTING_BLOCKS_METADATA); - assertBlocked(indicesAdmin().prepareGetMappings(), INDEX_METADATA_BLOCK); + assertBlocked(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT), INDEX_METADATA_BLOCK); } finally { disableIndexBlock("test", SETTING_BLOCKS_METADATA); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetMappingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetMappingsIT.java index 20e59fab3bd0c..023aa402b7337 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetMappingsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetMappingsIT.java @@ -42,7 +42,7 @@ protected Collection> nodePlugins() { public void testGetMappingsWhereThereAreNone() { createIndex("index"); - GetMappingsResponse response = indicesAdmin().prepareGetMappings().get(); + GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT).get(); assertThat(response.mappings().containsKey("index"), equalTo(true)); assertEquals(MappingMetadata.EMPTY_MAPPINGS, response.mappings().get("index")); } @@ -70,19 +70,19 @@ public void testSimpleGetMappings() throws Exception { assertThat(clusterHealth.isTimedOut(), equalTo(false)); // Get all mappings - GetMappingsResponse response = indicesAdmin().prepareGetMappings().get(); + GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT).get(); assertThat(response.mappings().size(), equalTo(2)); assertThat(response.mappings().get("indexa"), notNullValue()); assertThat(response.mappings().get("indexb"), notNullValue()); // Get all mappings, via wildcard support - response = indicesAdmin().prepareGetMappings("*").get(); + response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "*").get(); assertThat(response.mappings().size(), equalTo(2)); assertThat(response.mappings().get("indexa"), notNullValue()); assertThat(response.mappings().get("indexb"), notNullValue()); // Get mappings in indexa - response = indicesAdmin().prepareGetMappings("indexa").get(); + response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "indexa").get(); assertThat(response.mappings().size(), equalTo(1)); assertThat(response.mappings().get("indexa"), notNullValue()); } @@ -94,7 +94,7 @@ public void testGetMappingsWithBlocks() throws IOException { for (String block : Arrays.asList(SETTING_BLOCKS_READ, SETTING_BLOCKS_WRITE, SETTING_READ_ONLY)) { try { enableIndexBlock("test", block); - GetMappingsResponse response = indicesAdmin().prepareGetMappings().get(); + GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT).get(); assertThat(response.mappings().size(), equalTo(1)); assertNotNull(response.mappings().get("test")); } finally { @@ -104,7 +104,7 @@ public void testGetMappingsWithBlocks() throws IOException { try { enableIndexBlock("test", SETTING_BLOCKS_METADATA); - assertBlocked(indicesAdmin().prepareGetMappings(), INDEX_METADATA_BLOCK); + assertBlocked(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT), INDEX_METADATA_BLOCK); } finally { disableIndexBlock("test", SETTING_BLOCKS_METADATA); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java index 6f6e488d46b23..fa2598348a1ce 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java @@ -109,7 +109,7 @@ public void testUpdateMappingWithoutType() { assertThat(putMappingResponse.isAcknowledged(), equalTo(true)); - GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings("test").get(); + GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get(); assertThat(getMappingsResponse.mappings().get("test").source().toString(), equalTo(""" {"_doc":{"properties":{"body":{"type":"text"},"date":{"type":"integer"}}}}""")); } @@ -123,7 +123,7 @@ public void testUpdateMappingWithoutTypeMultiObjects() { assertThat(putMappingResponse.isAcknowledged(), equalTo(true)); - GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings("test").get(); + GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get(); assertThat(getMappingsResponse.mappings().get("test").source().toString(), equalTo(""" {"_doc":{"properties":{"date":{"type":"integer"}}}}""")); } @@ -215,7 +215,10 @@ public void testUpdateMappingConcurrently() throws Throwable { .get(); assertThat(response.isAcknowledged(), equalTo(true)); - GetMappingsResponse getMappingResponse = client2.admin().indices().prepareGetMappings(indexName).get(); + GetMappingsResponse getMappingResponse = client2.admin() + .indices() + .prepareGetMappings(TEST_REQUEST_TIMEOUT, indexName) + .get(); MappingMetadata mappings = getMappingResponse.getMappings().get(indexName); @SuppressWarnings("unchecked") Map properties = (Map) mappings.getSourceAsMap().get("properties"); @@ -284,7 +287,7 @@ private void assertConcreteMappingsOnAll(final String index, final String... fie * Waits for the given mapping type to exists on the master node. */ private void assertMappingOnMaster(final String index, final String... fieldNames) { - GetMappingsResponse response = indicesAdmin().prepareGetMappings(index).get(); + GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, index).get(); MappingMetadata mappings = response.getMappings().get(index); assertThat(mappings, notNullValue()); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequest.java index dd4114c947174..84789d8a2acfb 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequest.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.master.info.ClusterInfoRequest; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; @@ -21,7 +22,9 @@ public class GetMappingsRequest extends ClusterInfoRequest { - public GetMappingsRequest() {} + public GetMappingsRequest(TimeValue masterTimeout) { + super(masterTimeout); + } public GetMappingsRequest(StreamInput in) throws IOException { super(in); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequestBuilder.java index 3f54138581398..a12ba4f60c26a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequestBuilder.java @@ -11,13 +11,14 @@ import org.elasticsearch.action.support.master.info.ClusterInfoRequestBuilder; import org.elasticsearch.client.internal.ElasticsearchClient; +import org.elasticsearch.core.TimeValue; public class GetMappingsRequestBuilder extends ClusterInfoRequestBuilder< GetMappingsRequest, GetMappingsResponse, GetMappingsRequestBuilder> { - public GetMappingsRequestBuilder(ElasticsearchClient client, String... indices) { - super(client, GetMappingsAction.INSTANCE, new GetMappingsRequest().indices(indices)); + public GetMappingsRequestBuilder(ElasticsearchClient client, TimeValue masterTimeout, String... indices) { + super(client, GetMappingsAction.INSTANCE, new GetMappingsRequest(masterTimeout).indices(indices)); } } diff --git a/server/src/main/java/org/elasticsearch/action/support/master/info/ClusterInfoRequest.java b/server/src/main/java/org/elasticsearch/action/support/master/info/ClusterInfoRequest.java index 943b03588d4ba..634a103e9754a 100644 --- a/server/src/main/java/org/elasticsearch/action/support/master/info/ClusterInfoRequest.java +++ b/server/src/main/java/org/elasticsearch/action/support/master/info/ClusterInfoRequest.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.TimeValue; import java.io.IOException; @@ -27,11 +28,22 @@ public abstract class ClusterInfoRequest getMappings(GetMappingsRequest request) return execute(GetMappingsAction.INSTANCE, request); } - public GetMappingsRequestBuilder prepareGetMappings(String... indices) { - return new GetMappingsRequestBuilder(this, indices); + public GetMappingsRequestBuilder prepareGetMappings(TimeValue masterTimeout, String... indices) { + return new GetMappingsRequestBuilder(this, masterTimeout, indices); } public void getFieldMappings(GetFieldMappingsRequest request, ActionListener listener) { diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java index 27620fa750ea9..f9e02646041a6 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java @@ -13,7 +13,6 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.Strings; -import org.elasticsearch.core.TimeValue; import org.elasticsearch.http.HttpChannel; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; @@ -51,11 +50,9 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { final String[] indices = Strings.splitStringByCommaToArray(request.param("index")); - final GetMappingsRequest getMappingsRequest = new GetMappingsRequest(); + final GetMappingsRequest getMappingsRequest = new GetMappingsRequest(getMasterNodeTimeout(request)); getMappingsRequest.indices(indices); getMappingsRequest.indicesOptions(IndicesOptions.fromRequest(request, getMappingsRequest.indicesOptions())); - final TimeValue timeout = getMasterNodeTimeout(request); - getMappingsRequest.masterNodeTimeout(timeout); getMappingsRequest.local(request.paramAsBoolean("local", getMappingsRequest.local())); final HttpChannel httpChannel = request.getHttpChannel(); return channel -> new RestCancellableNodeClient(client, httpChannel).admin() diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java index 907a1a15721dc..4b0fac2cf2e0f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java @@ -54,7 +54,7 @@ public void putMappings() { } public void testGetMappings() { - GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings().get(); + GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT).get(); assertExpectedMappings(getMappingsResponse.mappings()); } @@ -71,7 +71,7 @@ public void testGetFieldMappings() { assertFieldMappings(mappings.get("filtered"), FILTERED_FLAT_FIELDS); // double check that submitting the filtered mappings to an unfiltered index leads to the same get field mappings output // as the one coming from a filtered index with same mappings - GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings("filtered").get(); + GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "filtered").get(); MappingMetadata filtered = getMappingsResponse.getMappings().get("filtered"); assertAcked(indicesAdmin().prepareCreate("test").setMapping(filtered.getSourceAsMap())); GetFieldMappingsResponse response = indicesAdmin().prepareGetFieldMappings("test").setFields("*").get(); @@ -98,7 +98,7 @@ public void testFieldCapabilities() { assertFieldCaps(filtered, filteredFields); // double check that submitting the filtered mappings to an unfiltered index leads to the same field_caps output // as the one coming from a filtered index with same mappings - GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings("filtered").get(); + GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "filtered").get(); MappingMetadata filteredMapping = getMappingsResponse.getMappings().get("filtered"); assertAcked(indicesAdmin().prepareCreate("test").setMapping(filteredMapping.getSourceAsMap())); FieldCapabilitiesResponse test = client().fieldCaps(new FieldCapabilitiesRequest().fields("*").indices("test")).actionGet(); @@ -155,7 +155,7 @@ private void assertExpectedMappings(Map mappings) { private void assertMappingsAreValid(Map sourceAsMap) { // check that the returned filtered mappings are still valid mappings by submitting them and retrieving them back assertAcked(indicesAdmin().prepareCreate("test").setMapping(sourceAsMap)); - GetMappingsResponse testMappingsResponse = indicesAdmin().prepareGetMappings("test").get(); + GetMappingsResponse testMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get(); assertEquals(1, testMappingsResponse.getMappings().size()); // the mappings are returned unfiltered for this index, yet they are the same as the previous ones that were returned filtered assertFiltered(testMappingsResponse.getMappings().get("test")); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NonDynamicFieldMapperTestCase.java b/server/src/test/java/org/elasticsearch/index/mapper/NonDynamicFieldMapperTestCase.java index 6ea41763cbcc2..b669c717719b0 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NonDynamicFieldMapperTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NonDynamicFieldMapperTestCase.java @@ -43,7 +43,7 @@ public void testCreateExplicitMappingSucceeds() throws Exception { """, getMapping()); var resp = client().admin().indices().prepareCreate("test").setMapping(mapping).get(); assertTrue(resp.isAcknowledged()); - var mappingsResp = client().admin().indices().prepareGetMappings("test").get(); + var mappingsResp = client().admin().indices().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get(); var mappingMetadata = mappingsResp.getMappings().get("test"); var fieldType = XContentMapValues.extractValue("properties.field.type", mappingMetadata.getSourceAsMap()); assertThat(fieldType, equalTo(getTypeName())); @@ -149,7 +149,7 @@ public void testCreateExplicitMappingInIndexTemplateSucceeds() throws Exception var resp = client().prepareIndex("test1").setSource("field", "hello world").get(); assertThat(resp.status(), equalTo(RestStatus.CREATED)); - var mappingsResp = client().admin().indices().prepareGetMappings("test1").get(); + var mappingsResp = client().admin().indices().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test1").get(); var mappingMetadata = mappingsResp.getMappings().get("test1"); var fieldType = XContentMapValues.extractValue("properties.field.type", mappingMetadata.getSourceAsMap()); assertThat(fieldType, equalTo(getTypeName())); diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java index 7ce55313aa771..cca8228557aba 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java @@ -488,7 +488,7 @@ public void testFollowerMappingIsUpdated() throws IOException { clusterStateRequest.indices(followerIndex); MappingMetadata mappingMetadata = followerClient().admin() .indices() - .prepareGetMappings("index2") + .prepareGetMappings(TEST_REQUEST_TIMEOUT, "index2") .get() .getMappings() .get("index2"); diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java index 7556d02ddb1cf..e50627822e315 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java @@ -371,7 +371,12 @@ public void testSyncMappings() throws Exception { } assertBusy(() -> assertHitCount(followerClient().prepareSearch("index2"), firstBatchNumDocs)); - MappingMetadata mappingMetadata = followerClient().admin().indices().prepareGetMappings("index2").get().getMappings().get("index2"); + MappingMetadata mappingMetadata = followerClient().admin() + .indices() + .prepareGetMappings(TEST_REQUEST_TIMEOUT, "index2") + .get() + .getMappings() + .get("index2"); assertThat(XContentMapValues.extractValue("properties.f.type", mappingMetadata.sourceAsMap()), equalTo("integer")); assertThat(XContentMapValues.extractValue("properties.k", mappingMetadata.sourceAsMap()), nullValue()); @@ -382,7 +387,12 @@ public void testSyncMappings() throws Exception { } assertBusy(() -> assertHitCount(followerClient().prepareSearch("index2"), firstBatchNumDocs + secondBatchNumDocs)); - mappingMetadata = followerClient().admin().indices().prepareGetMappings("index2").get().getMappings().get("index2"); + mappingMetadata = followerClient().admin() + .indices() + .prepareGetMappings(TEST_REQUEST_TIMEOUT, "index2") + .get() + .getMappings() + .get("index2"); assertThat(XContentMapValues.extractValue("properties.f.type", mappingMetadata.sourceAsMap()), equalTo("integer")); assertThat(XContentMapValues.extractValue("properties.k.type", mappingMetadata.sourceAsMap()), equalTo("long")); pauseFollow("index2"); @@ -410,7 +420,12 @@ public void testNoMappingDefined() throws Exception { assertBusy(() -> assertHitCount(followerClient().prepareSearch("index2"), 1)); pauseFollow("index2"); - MappingMetadata mappingMetadata = followerClient().admin().indices().prepareGetMappings("index2").get().getMappings().get("index2"); + MappingMetadata mappingMetadata = followerClient().admin() + .indices() + .prepareGetMappings(TEST_REQUEST_TIMEOUT, "index2") + .get() + .getMappings() + .get("index2"); assertThat(XContentMapValues.extractValue("properties.f.type", mappingMetadata.sourceAsMap()), equalTo("long")); assertThat(XContentMapValues.extractValue("properties.k", mappingMetadata.sourceAsMap()), nullValue()); } @@ -1188,7 +1203,7 @@ public void testUpdateAnalysisLeaderIndexSettings() throws Exception { assertThat(getSettingsResponse.getSetting("follower", "index.analysis.analyzer.my_analyzer.type"), equalTo("custom")); assertThat(getSettingsResponse.getSetting("follower", "index.analysis.analyzer.my_analyzer.tokenizer"), equalTo("keyword")); - GetMappingsRequest getMappingsRequest = new GetMappingsRequest(); + GetMappingsRequest getMappingsRequest = new GetMappingsRequest(TEST_REQUEST_TIMEOUT); getMappingsRequest.indices("follower"); GetMappingsResponse getMappingsResponse = followerClient().admin().indices().getMappings(getMappingsRequest).actionGet(); MappingMetadata mappingMetadata = getMappingsResponse.getMappings().get("follower"); diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotIT.java index 1bf52b663b30f..af7e223e1c5b9 100644 --- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotIT.java +++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotIT.java @@ -197,7 +197,7 @@ public void testSnapshotWithDanglingLocalSegment() throws Exception { } private static void assertMappings(String sourceIdx, boolean requireRouting, boolean useNested) throws IOException { - GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings(sourceIdx).get(); + GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings(TEST_REQUEST_TIMEOUT, sourceIdx).get(); MappingMetadata mapping = getMappingsResponse.getMappings().get(sourceIdx); String nested = useNested ? """ ,"incorrect":{"type":"object"},"nested":{"type":"nested","properties":{"value":{"type":"long"}}}""" : ""; diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java index 01a26a7a0b7bb..41d815c8206a7 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java @@ -285,7 +285,7 @@ protected void masterOperation( // At any point if there is an issue, delete the downsample index // 1. Extract source index mappings - final GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices(sourceIndexName); + final GetMappingsRequest getMappingsRequest = new GetMappingsRequest(request.masterNodeTimeout()).indices(sourceIndexName); getMappingsRequest.setParentTask(parentTask); client.admin().indices().getMappings(getMappingsRequest, listener.delegateFailureAndWrap((delegate, getMappingsResponse) -> { final Map sourceIndexMappings = getMappingsResponse.mappings() diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java index 33a436a1c7f1b..31b415270915b 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java @@ -1179,7 +1179,7 @@ private InternalAggregations aggregate(final String index, AggregationBuilder ag @SuppressWarnings("unchecked") private void assertDownsampleIndex(String sourceIndex, String downsampleIndex, DownsampleConfig config) throws Exception { // Retrieve field information for the metric fields - final GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(sourceIndex).get(); + final GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, sourceIndex).get(); final Map sourceIndexMappings = getMappingsResponse.mappings() .entrySet() .stream() @@ -1221,8 +1221,9 @@ private void assertDownsampleIndex(String sourceIndex, String downsampleIndex, D assertFieldMappings(config, metricFields, mappings); - GetMappingsResponse indexMappings = indicesAdmin().getMappings(new GetMappingsRequest().indices(downsampleIndex, sourceIndex)) - .actionGet(); + GetMappingsResponse indexMappings = indicesAdmin().getMappings( + new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(downsampleIndex, sourceIndex) + ).actionGet(); Map downsampleIndexProperties = (Map) indexMappings.mappings() .get(downsampleIndex) .sourceAsMap() diff --git a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceActionIT.java b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceActionIT.java index b460c6abfeee4..f1b14658c6b8e 100644 --- a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceActionIT.java +++ b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceActionIT.java @@ -110,7 +110,8 @@ public void testMappingsCopiedFromSource() { client().execute(CreateIndexFromSourceAction.INSTANCE, new CreateIndexFromSourceAction.Request(sourceIndex, destIndex)) ); - var mappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest().indices(sourceIndex, destIndex)).actionGet(); + var mappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(sourceIndex, destIndex)) + .actionGet(); Map mappings = mappingsResponse.mappings(); var destMappings = mappings.get(destIndex).sourceAsMap(); var sourceMappings = mappings.get(sourceIndex).sourceAsMap(); @@ -224,7 +225,7 @@ public void testMappingsOverridden() { ) ); - var mappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest().indices(destIndex)).actionGet(); + var mappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(destIndex)).actionGet(); Map mappings = mappingsResponse.mappings(); var destMappings = mappings.get(destIndex).sourceAsMap(); diff --git a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java index 46af8ab2fb4c2..71b2a2a43b343 100644 --- a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java +++ b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java @@ -213,7 +213,8 @@ public void testMappingsAddedToDestIndex() throws Exception { .actionGet() .getDestIndex(); - var mappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest().indices(sourceIndex, destIndex)).actionGet(); + var mappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(sourceIndex, destIndex)) + .actionGet(); Map mappings = mappingsResponse.mappings(); var destMappings = mappings.get(destIndex).sourceAsMap(); var sourceMappings = mappings.get(sourceIndex).sourceAsMap(); @@ -319,7 +320,8 @@ public void testSettingsAndMappingsFromTemplate() throws IOException { // verify mappings from templates copied to dest index { - var mappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest().indices(sourceIndex, destIndex)).actionGet(); + var mappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(sourceIndex, destIndex)) + .actionGet(); var destMappings = mappingsResponse.mappings().get(destIndex).sourceAsMap(); var sourceMappings = mappingsResponse.mappings().get(sourceIndex).sourceAsMap(); assertEquals(sourceMappings, destMappings); diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java index ff6c3c8444783..7ce0fbe760644 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java @@ -255,7 +255,7 @@ public void testMultipleSimultaneousJobCreations() { // Assert that the mappings contain all the additional fields: field1, field2, field3, etc. String sharedResultsIndex = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT; - GetMappingsRequest request = new GetMappingsRequest().indices(sharedResultsIndex); + GetMappingsRequest request = new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(sharedResultsIndex); GetMappingsResponse response = client().execute(GetMappingsAction.INSTANCE, request).actionGet(); Map indexMappings = response.getMappings(); assertNotNull(indexMappings); @@ -506,7 +506,7 @@ public void testGetDataCountsModelSizeAndTimingStatsWithSomeDocs() throws Except } private Map getIndexMappingProperties(String index) { - GetMappingsRequest request = new GetMappingsRequest().indices(index); + GetMappingsRequest request = new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(index); GetMappingsResponse response = client().execute(GetMappingsAction.INSTANCE, request).actionGet(); Map indexMappings = response.getMappings(); assertNotNull(indexMappings); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java index 759538b4cdc63..dea5e891bd8dd 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java @@ -223,7 +223,7 @@ public void onFailure(Exception e) { ); // Get start context - getStartContext(request.getId(), task, startContextListener); + getStartContext(request.getId(), task, startContextListener, request.masterNodeTimeout()); } private void estimateMemoryUsageAndUpdateMemoryTracker(StartContext startContext, ActionListener listener) { @@ -264,7 +264,7 @@ private void estimateMemoryUsageAndUpdateMemoryTracker(StartContext startContext } - private void getStartContext(String id, Task task, ActionListener finalListener) { + private void getStartContext(String id, Task task, ActionListener finalListener, TimeValue masterTimeout) { ParentTaskAssigningClient parentTaskClient = new ParentTaskAssigningClient(client, task.getParentTaskId()); @@ -320,6 +320,7 @@ private void getStartContext(String id, Task task, ActionListener .andThen( (l, startContext) -> MappingsMerger.mergeMappings( parentTaskClient, + masterTimeout, startContext.config.getHeaders(), startContext.config.getSource(), l.map(ignored -> startContext) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndex.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndex.java index ed12f54ab86b5..ba69c8733dd13 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndex.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndex.java @@ -38,6 +38,7 @@ import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsDest; import org.elasticsearch.xpack.core.ml.dataframe.analyses.RequiredField; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.ml.MachineLearning; import java.time.Clock; import java.util.Arrays; @@ -147,6 +148,7 @@ private static void prepareCreateIndexRequest( final Settings settings = settings(settingsResponse, destIndexAllowedSettings); MappingsMerger.mergeMappings( client, + MachineLearning.HARD_CODED_MACHINE_LEARNING_MASTER_NODE_TIMEOUT, config.getHeaders(), config.getSource(), delegate.delegateFailureAndWrap( diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/MappingsMerger.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/MappingsMerger.java index 8fee0160cf92e..817020d788554 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/MappingsMerger.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/MappingsMerger.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.MappingMetadata; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsSource; @@ -34,6 +35,7 @@ private MappingsMerger() {} public static void mergeMappings( Client client, + TimeValue masterTimeout, Map headers, DataFrameAnalyticsSource source, ActionListener listener @@ -43,7 +45,7 @@ public static void mergeMappings( listener::onFailure ); - GetMappingsRequest getMappingsRequest = new GetMappingsRequest(); + GetMappingsRequest getMappingsRequest = new GetMappingsRequest(masterTimeout); getMappingsRequest.indices(source.getIndex()); ClientHelper.executeWithHeadersAsync(headers, ML_ORIGIN, client, GetMappingsAction.INSTANCE, getMappingsRequest, mappingsListener); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java index 51b3e0b55d75b..23b12a23e7495 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java @@ -126,6 +126,7 @@ import org.elasticsearch.xpack.core.ml.stats.StatsAccumulator; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.core.security.support.Exceptions; +import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.ml.job.categorization.GrokPatternCreator; import org.elasticsearch.xpack.ml.job.persistence.InfluencersQueryBuilder.InfluencersQuery; import org.elasticsearch.xpack.ml.job.process.autodetect.params.AutodetectParams; @@ -392,7 +393,10 @@ private void getLatestIndexMappingsAndAddTerms(String indexName, Collection startPersistentTask(job, listener, persistentTasksService), e -> { if (e instanceof ResourceAlreadyExistsException) { logger.debug("Rolled index already exists for rollup job [" + job.getConfig().getId() + "], updating metadata."); - updateMapping(job, listener, persistentTasksService, client, logger); + updateMapping(job, listener, persistentTasksService, client, logger, request.masterNodeTimeout()); } else { String msg = "Could not create index for rollup job [" + job.getConfig().getId() + "]"; logger.error(msg); @@ -249,7 +249,8 @@ static void updateMapping( ActionListener listener, PersistentTasksService persistentTasksService, Client client, - Logger logger + Logger logger, + TimeValue masterTimeout ) { final String indexName = job.getConfig().getRollupIndex(); @@ -304,7 +305,7 @@ static void updateMapping( ); }; - GetMappingsRequest request = new GetMappingsRequest(); + GetMappingsRequest request = new GetMappingsRequest(masterTimeout); client.execute(GetMappingsAction.INSTANCE, request, ActionListener.wrap(getMappingResponseHandler, e -> { String msg = "Could not update mappings for rollup job [" + job.getConfig().getId() + "]"; logger.error(msg); diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobStateMachineTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobStateMachineTests.java index ee8b4c79d1893..fed2439e513c8 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobStateMachineTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobStateMachineTests.java @@ -166,7 +166,14 @@ public void testGetMappingFails() { return null; }).when(client).execute(eq(GetMappingsAction.INSTANCE), any(GetMappingsRequest.class), requestCaptor.capture()); - TransportPutRollupJobAction.updateMapping(job, testListener, mock(PersistentTasksService.class), client, logger); + TransportPutRollupJobAction.updateMapping( + job, + testListener, + mock(PersistentTasksService.class), + client, + logger, + TEST_REQUEST_TIMEOUT + ); verify(client).execute(eq(GetMappingsAction.INSTANCE), any(GetMappingsRequest.class), any()); } @@ -201,7 +208,14 @@ public void testNoMetadataInMapping() { return null; }).when(client).execute(eq(GetMappingsAction.INSTANCE), any(GetMappingsRequest.class), requestCaptor.capture()); - TransportPutRollupJobAction.updateMapping(job, testListener, mock(PersistentTasksService.class), client, logger); + TransportPutRollupJobAction.updateMapping( + job, + testListener, + mock(PersistentTasksService.class), + client, + logger, + TEST_REQUEST_TIMEOUT + ); verify(client).execute(eq(GetMappingsAction.INSTANCE), any(GetMappingsRequest.class), any()); } @@ -238,7 +252,14 @@ public void testMetadataButNotRollup() { return null; }).when(client).execute(eq(GetMappingsAction.INSTANCE), any(GetMappingsRequest.class), requestCaptor.capture()); - TransportPutRollupJobAction.updateMapping(job, testListener, mock(PersistentTasksService.class), client, logger); + TransportPutRollupJobAction.updateMapping( + job, + testListener, + mock(PersistentTasksService.class), + client, + logger, + TEST_REQUEST_TIMEOUT + ); verify(client).execute(eq(GetMappingsAction.INSTANCE), any(GetMappingsRequest.class), any()); } @@ -272,7 +293,14 @@ public void testJobAlreadyInMapping() { return null; }).when(client).execute(eq(GetMappingsAction.INSTANCE), any(GetMappingsRequest.class), requestCaptor.capture()); - TransportPutRollupJobAction.updateMapping(job, testListener, mock(PersistentTasksService.class), client, logger); + TransportPutRollupJobAction.updateMapping( + job, + testListener, + mock(PersistentTasksService.class), + client, + logger, + TEST_REQUEST_TIMEOUT + ); verify(client).execute(eq(GetMappingsAction.INSTANCE), any(GetMappingsRequest.class), any()); } @@ -318,7 +346,14 @@ public void testAddJobToMapping() { return null; }).when(client).execute(eq(TransportPutMappingAction.TYPE), any(PutMappingRequest.class), requestCaptor2.capture()); - TransportPutRollupJobAction.updateMapping(job, testListener, mock(PersistentTasksService.class), client, logger); + TransportPutRollupJobAction.updateMapping( + job, + testListener, + mock(PersistentTasksService.class), + client, + logger, + TEST_REQUEST_TIMEOUT + ); verify(client).execute(eq(GetMappingsAction.INSTANCE), any(GetMappingsRequest.class), any()); verify(client).execute(eq(TransportPutMappingAction.TYPE), any(PutMappingRequest.class), any()); } diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java index 57d18abaf1a92..f7361480f0a51 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java @@ -287,28 +287,28 @@ public void testGetMappingsIsFiltered() { { GetMappingsResponse getMappingsResponse = client().filterWithHeader( Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD)) - ).admin().indices().prepareGetMappings("test").get(); + ).admin().indices().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get(); assertExpectedMetadataFields(getMappingsResponse.getMappings(), "field1"); } { GetMappingsResponse getMappingsResponse = client().filterWithHeader( Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD)) - ).admin().indices().prepareGetMappings("test").get(); + ).admin().indices().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get(); assertExpectedMetadataFields(getMappingsResponse.getMappings(), "field2"); } { GetMappingsResponse getMappingsResponse = client().filterWithHeader( Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user3", USERS_PASSWD)) - ).admin().indices().prepareGetMappings("test").get(); + ).admin().indices().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get(); assertExpectedMetadataFields(getMappingsResponse.getMappings(), "field1"); } { GetMappingsResponse getMappingsResponse = client().filterWithHeader( Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user4", USERS_PASSWD)) - ).admin().indices().prepareGetMappings("test").get(); + ).admin().indices().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get(); assertExpectedMetadataFields(getMappingsResponse.getMappings(), "field1", "field2"); } } diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java index bc730b5695c19..b501ba69d673b 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java @@ -166,7 +166,7 @@ public void testGetMappings() throws Exception { GetMappingsResponse response = client().filterWithHeader( singletonMap("Authorization", UsernamePasswordToken.basicAuthHeaderValue("kibana_user", USERS_PASSWD)) - ).admin().indices().prepareGetMappings("logstash-*").get(); + ).admin().indices().prepareGetMappings(TEST_REQUEST_TIMEOUT, "logstash-*").get(); Map mappingsMap = response.getMappings(); assertNotNull(mappingsMap); assertNotNull(mappingsMap.get(index)); diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateHttpMappingsTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateHttpMappingsTests.java index ffac36846414e..ea9d76fb643fb 100644 --- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateHttpMappingsTests.java +++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateHttpMappingsTests.java @@ -172,7 +172,8 @@ public void testExceptionMapping() throws Exception { // ensure that enabled is set to false List indexed = new ArrayList<>(); - GetMappingsResponse mappingsResponse = indicesAdmin().prepareGetMappings(HistoryStoreField.INDEX_PREFIX + "*").get(); + GetMappingsResponse mappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, HistoryStoreField.INDEX_PREFIX + "*") + .get(); for (MappingMetadata mapping : mappingsResponse.getMappings().values()) { Map docMapping = mapping.getSourceAsMap(); if (abortAtInput) { diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateTimeMappingsTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateTimeMappingsTests.java index c78315383df2f..34d38dfde064d 100644 --- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateTimeMappingsTests.java +++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateTimeMappingsTests.java @@ -46,7 +46,7 @@ public void testTimeFields() throws Exception { assertWatchWithMinimumActionsCount("_id", ExecutionState.EXECUTED, 1); assertBusy(() -> { - GetMappingsResponse mappingsResponse = indicesAdmin().prepareGetMappings() + GetMappingsResponse mappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT) .setIndicesOptions(IndicesOptions.strictExpandHidden()) .get(); assertThat(mappingsResponse, notNullValue()); diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java index 1bcdd060994ce..732b0800b4316 100644 --- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java +++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java @@ -120,7 +120,7 @@ public void testFailedInputResultWithDotsInFieldNameGetsStored() throws Exceptio assertBusy(() -> { assertHitCount(getWatchHistory(), 1); }); // as fields with dots are allowed in 5.0 again, the mapping must be checked in addition - GetMappingsResponse response = indicesAdmin().prepareGetMappings(".watcher-history*").get(); + GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, ".watcher-history*").get(); XContentSource source = new XContentSource( response.getMappings().values().iterator().next().source().uncompressed(), XContentType.JSON @@ -162,7 +162,7 @@ public void testPayloadInputWithDotsInFieldNameWorks() throws Exception { assertBusy(() -> { assertHitCount(getWatchHistory(), 1); }); // as fields with dots are allowed in 5.0 again, the mapping must be checked in addition - GetMappingsResponse response = indicesAdmin().prepareGetMappings(".watcher-history*").get(); + GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, ".watcher-history*").get(); XContentSource source = new XContentSource( response.getMappings().values().iterator().next().source().uncompressed(), XContentType.JSON @@ -225,7 +225,7 @@ public void testThatHistoryContainsStatus() throws Exception { assertBusy(() -> { // also ensure that the status field is disabled in the watch history - GetMappingsResponse response = indicesAdmin().prepareGetMappings(".watcher-history*").get(); + GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, ".watcher-history*").get(); XContentSource mappingSource = new XContentSource( response.getMappings().values().iterator().next().source().uncompressed(), XContentType.JSON From 987da8bd967fb5a5c806f96830485403252ee43c Mon Sep 17 00:00:00 2001 From: Niels Bauman <33722607+nielsbauman@users.noreply.github.com> Date: Tue, 14 Jan 2025 12:40:30 +1000 Subject: [PATCH 05/31] Remove trappy timeouts from `ExplainLifecycleRequest` (#120036) Relates #107984 --- .../existence/FrozenExistenceDeciderIT.java | 2 +- .../core/ilm/ExplainLifecycleRequest.java | 5 +-- .../ilm/ExplainLifecycleRequestTests.java | 4 +-- .../ClusterStateWaitThresholdBreachTests.java | 10 +++--- ...ataStreamAndIndexLifecycleMixingTests.java | 36 ++++++++++++------- .../xpack/ilm/DataTiersMigrationsTests.java | 12 +++---- .../xpack/ilm/ILMMultiNodeIT.java | 6 ++-- .../ilm/ILMMultiNodeWithCCRDisabledIT.java | 6 ++-- .../IndexLifecycleInitialisationTests.java | 2 +- .../action/RestExplainLifecycleAction.java | 3 +- 10 files changed, 52 insertions(+), 34 deletions(-) diff --git a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/existence/FrozenExistenceDeciderIT.java b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/existence/FrozenExistenceDeciderIT.java index df1022e45c5a5..2357b80a402b3 100644 --- a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/existence/FrozenExistenceDeciderIT.java +++ b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/existence/FrozenExistenceDeciderIT.java @@ -123,7 +123,7 @@ public void testZeroToOne() throws Exception { assertBusy(() -> { ExplainLifecycleResponse response = client().execute( ExplainLifecycleAction.INSTANCE, - new ExplainLifecycleRequest().indices(INDEX_NAME) + new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(INDEX_NAME) ).actionGet(); IndexLifecycleExplainResponse indexResponse = response.getIndexResponses().get(INDEX_NAME); assertNotNull(indexResponse); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ExplainLifecycleRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ExplainLifecycleRequest.java index 5c607335bff39..beff1e6ab39de 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ExplainLifecycleRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ExplainLifecycleRequest.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.support.master.info.ClusterInfoRequest; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.TimeValue; import java.io.IOException; import java.util.Arrays; @@ -27,8 +28,8 @@ public class ExplainLifecycleRequest extends ClusterInfoRequest onlyManaged = onlyManaged == false; default -> throw new AssertionError("Illegal randomisation branch"); } - ExplainLifecycleRequest newRequest = new ExplainLifecycleRequest(); + ExplainLifecycleRequest newRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT); newRequest.indices(indices); newRequest.indicesOptions(indicesOptions); newRequest.onlyErrors(onlyErrors); diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ClusterStateWaitThresholdBreachTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ClusterStateWaitThresholdBreachTests.java index f25028824b56e..5d4b837dc4d1d 100644 --- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ClusterStateWaitThresholdBreachTests.java +++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ClusterStateWaitThresholdBreachTests.java @@ -108,7 +108,7 @@ public void testWaitInShrunkShardsAllocatedExceedsThreshold() throws Exception { String[] firstAttemptShrinkIndexName = new String[1]; assertBusy(() -> { - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(managedIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(managedIndex); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex); @@ -118,7 +118,7 @@ public void testWaitInShrunkShardsAllocatedExceedsThreshold() throws Exception { // let's check ILM for the managed index is waiting in the `shrunk-shards-allocated` step assertBusy(() -> { - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(managedIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(managedIndex); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex); @@ -160,7 +160,7 @@ public void onFailure(Exception e) { String[] secondCycleShrinkIndexName = new String[1]; assertBusy(() -> { - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(managedIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(managedIndex); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex); @@ -180,7 +180,9 @@ public void onFailure(Exception e) { // situation and allow for shrink to complete by reducing the number of shards for the shrunk index to 0 setReplicaCount(0, secondCycleShrinkIndexName[0]); assertBusy(() -> { - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(secondCycleShrinkIndexName[0]); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( + secondCycleShrinkIndexName[0] + ); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses() .get(secondCycleShrinkIndexName[0]); diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataStreamAndIndexLifecycleMixingTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataStreamAndIndexLifecycleMixingTests.java index 21924634ff6ab..d9cc4de344a39 100644 --- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataStreamAndIndexLifecycleMixingTests.java +++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataStreamAndIndexLifecycleMixingTests.java @@ -149,7 +149,10 @@ public void testIndexTemplateSwapsILMForDataStreamLifecycle() throws Exception { List backingIndices = getBackingIndices(dataStreamName); String firstGenerationIndex = backingIndices.get(0); String secondGenerationIndex = backingIndices.get(1); - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(firstGenerationIndex, secondGenerationIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( + firstGenerationIndex, + secondGenerationIndex + ); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse firstGenerationExplain = explainResponse.getIndexResponses().get(firstGenerationIndex); @@ -193,7 +196,7 @@ public void testIndexTemplateSwapsILMForDataStreamLifecycle() throws Exception { String firstGenerationIndex = backingIndices.get(0); String secondGenerationIndex = backingIndices.get(1); String writeIndex = backingIndices.get(2); - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices( + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( firstGenerationIndex, secondGenerationIndex, writeIndex @@ -270,7 +273,7 @@ public void testIndexTemplateSwapsILMForDataStreamLifecycle() throws Exception { String secondGenerationIndex = backingIndices.get(1); String thirdGenerationIndex = backingIndices.get(2); String writeIndex = backingIndices.get(3); - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices( + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( firstGenerationIndex, secondGenerationIndex, thirdGenerationIndex, @@ -348,7 +351,10 @@ public void testUpdateIndexTemplateFromILMtoBothILMAndDataStreamLifecycle() thro List backingIndices = getBackingIndices(dataStreamName); String firstGenerationIndex = backingIndices.get(0); String secondGenerationIndex = backingIndices.get(1); - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(firstGenerationIndex, secondGenerationIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( + firstGenerationIndex, + secondGenerationIndex + ); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse firstGenerationExplain = explainResponse.getIndexResponses().get(firstGenerationIndex); @@ -401,7 +407,7 @@ public void testUpdateIndexTemplateFromILMtoBothILMAndDataStreamLifecycle() thro String firstGenerationIndex = backingIndices.get(0); String secondGenerationIndex = backingIndices.get(1); String thirdGenerationIndex = backingIndices.get(2); - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices( + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( firstGenerationIndex, secondGenerationIndex, thirdGenerationIndex @@ -465,7 +471,7 @@ public void testUpdateIndexTemplateFromILMtoBothILMAndDataStreamLifecycle() thro String secondGenerationIndex = backingIndices.get(1); String thirdGenerationIndex = backingIndices.get(2); String writeIndex = backingIndices.get(3); - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices( + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( firstGenerationIndex, secondGenerationIndex, thirdGenerationIndex, @@ -543,7 +549,10 @@ public void testUpdateIndexTemplateToDataStreamLifecyclePreference() throws Exce List backingIndices = getBackingIndices(dataStreamName); String firstGenerationIndex = backingIndices.get(0); String secondGenerationIndex = backingIndices.get(1); - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(firstGenerationIndex, secondGenerationIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( + firstGenerationIndex, + secondGenerationIndex + ); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse firstGenerationExplain = explainResponse.getIndexResponses().get(firstGenerationIndex); @@ -591,7 +600,7 @@ public void testUpdateIndexTemplateToDataStreamLifecyclePreference() throws Exce String firstGenerationIndex = backingIndices.get(0); String secondGenerationIndex = backingIndices.get(1); String writeIndex = backingIndices.get(2); - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices( + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( firstGenerationIndex, secondGenerationIndex, writeIndex @@ -663,7 +672,7 @@ public void testUpdateIndexTemplateToDataStreamLifecyclePreference() throws Exce String secondGenerationIndex = backingIndices.get(1); String thirdGenerationIndex = backingIndices.get(2); String writeIndex = backingIndices.get(3); - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices( + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( firstGenerationIndex, secondGenerationIndex, thirdGenerationIndex, @@ -720,7 +729,7 @@ public void testUpdateIndexTemplateToDataStreamLifecyclePreference() throws Exce String secondGenerationIndex = backingIndices.get(1); String thirdGenerationIndex = backingIndices.get(2); String writeIndex = backingIndices.get(3); - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices( + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( firstGenerationIndex, secondGenerationIndex, thirdGenerationIndex, @@ -846,7 +855,7 @@ public void testUpdateIndexTemplateToMigrateFromDataStreamLifecycleToIlm() throw assertThat(index.isManagedByLifecycle(), is(true)); } - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(writeIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(writeIndex); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse writeIndexExplain = explainResponse.getIndexResponses().get(writeIndex); @@ -875,7 +884,10 @@ public void testUpdateIndexTemplateToMigrateFromDataStreamLifecycleToIlm() throw String thirdGenerationIndex = backingIndices.get(2); String writeIndex = backingIndices.get(3); - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(thirdGenerationIndex, writeIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( + thirdGenerationIndex, + writeIndex + ); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse thirdGenerationExplain = explainResponse.getIndexResponses().get(thirdGenerationIndex); diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataTiersMigrationsTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataTiersMigrationsTests.java index 6d409bf474cfc..d483401931830 100644 --- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataTiersMigrationsTests.java +++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataTiersMigrationsTests.java @@ -116,7 +116,7 @@ public void testIndexDataTierMigration() throws Exception { assertTrue(res.isAcknowledged()); assertBusy(() -> { - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(managedIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(managedIndex); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex); @@ -127,7 +127,7 @@ public void testIndexDataTierMigration() throws Exception { logger.info("starting a warm data node"); internalCluster().startNode(warmNode(Settings.EMPTY)); assertBusy(() -> { - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(managedIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(managedIndex); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex); @@ -140,7 +140,7 @@ public void testIndexDataTierMigration() throws Exception { // wait for lifecycle to complete in the cold phase after the index has been migrated to the cold node assertBusy(() -> { - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(managedIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(managedIndex); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex); @@ -177,7 +177,7 @@ public void testUserOptsOutOfTierMigration() throws Exception { assertTrue(res.isAcknowledged()); assertBusy(() -> { - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(managedIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(managedIndex); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex); @@ -197,7 +197,7 @@ public void testUserOptsOutOfTierMigration() throws Exception { // 1. start another cold node so both the primary and replica can relocate to the cold nodes // 2. remove the tier routing setting from the index again (we're doing this below) assertBusy(() -> { - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(managedIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(managedIndex); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex); @@ -210,7 +210,7 @@ public void testUserOptsOutOfTierMigration() throws Exception { // wait for lifecycle to complete in the cold phase assertBusy(() -> { - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(managedIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(managedIndex); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex); diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeIT.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeIT.java index 2c4c1c9e20bb6..3f92abc7e3da2 100644 --- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeIT.java +++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeIT.java @@ -99,8 +99,10 @@ public void testShrinkOnTiers() throws Exception { prepareIndex(index).setCreate(true).setId("1").setSource("@timestamp", "2020-09-09").get(); assertBusy(() -> { - ExplainLifecycleResponse explain = client().execute(ExplainLifecycleAction.INSTANCE, new ExplainLifecycleRequest().indices("*")) - .get(); + ExplainLifecycleResponse explain = client().execute( + ExplainLifecycleAction.INSTANCE, + new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices("*") + ).get(); logger.info("--> explain: {}", Strings.toString(explain)); String backingIndexName = DataStream.getDefaultBackingIndexName(index, 1); diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeWithCCRDisabledIT.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeWithCCRDisabledIT.java index b91a309a23ae5..c695dc8f87b68 100644 --- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeWithCCRDisabledIT.java +++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeWithCCRDisabledIT.java @@ -99,8 +99,10 @@ public void testShrinkOnTiers() throws Exception { prepareIndex(index).setCreate(true).setId("1").setSource("@timestamp", "2020-09-09").get(); assertBusy(() -> { - ExplainLifecycleResponse explain = client().execute(ExplainLifecycleAction.INSTANCE, new ExplainLifecycleRequest().indices("*")) - .get(); + ExplainLifecycleResponse explain = client().execute( + ExplainLifecycleAction.INSTANCE, + new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices("*") + ).get(); logger.info("--> explain: {}", Strings.toString(explain)); String backingIndexName = DataStream.getDefaultBackingIndexName(index, 1); diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java index 644f88dc533b9..8999adc08f97b 100644 --- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java +++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java @@ -388,7 +388,7 @@ public void testExplainParseOriginationDate() throws Exception { private IndexLifecycleExplainResponse executeExplainRequestAndGetTestIndexResponse(String indexName) throws ExecutionException, InterruptedException { - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); assertThat(explainResponse.getIndexResponses().size(), equalTo(1)); return explainResponse.getIndexResponses().get(indexName); diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestExplainLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestExplainLifecycleAction.java index 195f989eab055..3950edf648e18 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestExplainLifecycleAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestExplainLifecycleAction.java @@ -36,12 +36,11 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) { String[] indexes = Strings.splitStringByCommaToArray(restRequest.param("index")); - ExplainLifecycleRequest explainLifecycleRequest = new ExplainLifecycleRequest(); + ExplainLifecycleRequest explainLifecycleRequest = new ExplainLifecycleRequest(getMasterNodeTimeout(restRequest)); explainLifecycleRequest.indices(indexes); explainLifecycleRequest.indicesOptions(IndicesOptions.fromRequest(restRequest, IndicesOptions.strictExpandOpen())); explainLifecycleRequest.onlyManaged(restRequest.paramAsBoolean("only_managed", false)); explainLifecycleRequest.onlyErrors(restRequest.paramAsBoolean("only_errors", false)); - explainLifecycleRequest.masterNodeTimeout(getMasterNodeTimeout(restRequest)); return channel -> client.execute(ExplainLifecycleAction.INSTANCE, explainLifecycleRequest, new RestToXContentListener<>(channel)); } } From 27a9c4d911c4308755e3cb8139599b145284d357 Mon Sep 17 00:00:00 2001 From: Niels Bauman <33722607+nielsbauman@users.noreply.github.com> Date: Tue, 14 Jan 2025 12:41:05 +1000 Subject: [PATCH 06/31] Run template simulation actions on local node (#120038) The actions `TransportSimulateTemplateAction` and `TransportSimulateIndexTemplateAction` solely need the cluster state, they can run on any node. Additionally, they need to be cancellable to avoid doing unnecessary work after a client failure or timeout. As a drive-by, this removes more usages of the trappy default master node timeout. --- docs/changelog/120038.yaml | 5 ++ .../http/RestActionCancellationIT.java | 17 ++++++ .../post/SimulateIndexTemplateRequest.java | 33 ++++++------ .../post/SimulateIndexTemplateResponse.java | 30 +++-------- .../template/post/SimulateTemplateAction.java | 53 +++++++------------ .../TransportSimulateIndexTemplateAction.java | 35 +++++++----- .../post/TransportSimulateTemplateAction.java | 35 +++++++----- .../RestSimulateIndexTemplateAction.java | 9 ++-- .../indices/RestSimulateTemplateAction.java | 14 +++-- .../SimulateIndexTemplateRequestTests.java | 34 ++---------- .../post/SimulateTemplateRequestTests.java | 38 ++----------- 11 files changed, 132 insertions(+), 171 deletions(-) create mode 100644 docs/changelog/120038.yaml diff --git a/docs/changelog/120038.yaml b/docs/changelog/120038.yaml new file mode 100644 index 0000000000000..fe3a2ccccc095 --- /dev/null +++ b/docs/changelog/120038.yaml @@ -0,0 +1,5 @@ +pr: 120038 +summary: Run template simulation actions on local node +area: Ingest Node +type: enhancement +issues: [] diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestActionCancellationIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestActionCancellationIT.java index ce514c5f1b1e7..0af12a41fa979 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestActionCancellationIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestActionCancellationIT.java @@ -18,6 +18,8 @@ import org.elasticsearch.action.admin.indices.template.get.GetComponentTemplateAction; import org.elasticsearch.action.admin.indices.template.get.GetComposableIndexTemplateAction; import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesAction; +import org.elasticsearch.action.admin.indices.template.post.SimulateIndexTemplateAction; +import org.elasticsearch.action.admin.indices.template.post.SimulateTemplateAction; import org.elasticsearch.action.support.CancellableActionTestPlugin; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.RefCountingListener; @@ -81,6 +83,21 @@ public void testGetComposableTemplateCancellation() { runRestActionCancellationTest(new Request(HttpGet.METHOD_NAME, "/_index_template"), GetComposableIndexTemplateAction.NAME); } + public void testSimulateTemplateCancellation() { + runRestActionCancellationTest( + new Request(HttpPost.METHOD_NAME, "/_index_template/_simulate/random_index_template"), + SimulateTemplateAction.NAME + ); + } + + public void testSimulateIndexTemplateCancellation() { + createIndex("test"); + runRestActionCancellationTest( + new Request(HttpPost.METHOD_NAME, "/_index_template/_simulate_index/test"), + SimulateIndexTemplateAction.NAME + ); + } + private void runRestActionCancellationTest(Request request, String actionName) { final var node = usually() ? internalCluster().getRandomNodeName() : internalCluster().startCoordinatingOnlyNode(Settings.EMPTY); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequest.java index ce29d65ececf9..003be58d19554 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequest.java @@ -12,16 +12,20 @@ import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; -import org.elasticsearch.action.support.master.MasterNodeReadRequest; +import org.elasticsearch.action.support.local.LocalClusterStateRequest; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; import java.io.IOException; +import java.util.Map; import java.util.Objects; -public class SimulateIndexTemplateRequest extends MasterNodeReadRequest { +public class SimulateIndexTemplateRequest extends LocalClusterStateRequest { private String indexName; @@ -30,14 +34,18 @@ public class SimulateIndexTemplateRequest extends MasterNodeReadRequest headers) { + return new CancellableTask(id, type, action, "", parentTaskId, headers); + } + public String getIndexName() { return indexName; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java index a521dac60e96a..1a04b6e4d7633 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java @@ -12,13 +12,11 @@ import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.admin.indices.rollover.RolloverConfiguration; -import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; import org.elasticsearch.cluster.metadata.ResettableValue; import org.elasticsearch.cluster.metadata.Template; -import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.util.Maps; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.UpdateForV10; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; @@ -67,27 +65,11 @@ public RolloverConfiguration getRolloverConfiguration() { return rolloverConfiguration; } - public SimulateIndexTemplateResponse(StreamInput in) throws IOException { - super(in); - resolvedTemplate = in.readOptionalWriteable(Template::new); - if (in.readBoolean()) { - int overlappingTemplatesCount = in.readInt(); - overlappingTemplates = Maps.newMapWithExpectedSize(overlappingTemplatesCount); - for (int i = 0; i < overlappingTemplatesCount; i++) { - String templateName = in.readString(); - overlappingTemplates.put(templateName, in.readStringCollectionAsList()); - } - } else { - this.overlappingTemplates = null; - } - rolloverConfiguration = in.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X) - ? in.readOptionalWriteable(RolloverConfiguration::new) - : null; - if (in.getTransportVersion().between(TransportVersions.V_8_14_0, TransportVersions.V_8_16_0)) { - in.readOptionalWriteable(DataStreamGlobalRetention::read); - } - } - + /** + * NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC we must remain able to write these responses until + * we no longer need to support calling this action remotely. + */ + @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT) @Override public void writeTo(StreamOutput out) throws IOException { out.writeOptionalWriteable(resolvedTemplate); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateAction.java index 75cc72416a854..15015b910767e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateAction.java @@ -14,12 +14,16 @@ import org.elasticsearch.action.ActionType; import org.elasticsearch.action.ValidateActions; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; -import org.elasticsearch.action.support.master.MasterNodeReadRequest; +import org.elasticsearch.action.support.local.LocalClusterStateRequest; import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; import java.io.IOException; +import java.util.Map; import java.util.Objects; /** @@ -35,7 +39,7 @@ private SimulateTemplateAction() { super(NAME); } - public static class Request extends MasterNodeReadRequest { + public static class Request extends LocalClusterStateRequest { @Nullable private String templateName; @@ -44,26 +48,15 @@ public static class Request extends MasterNodeReadRequest { private TransportPutComposableIndexTemplateAction.Request indexTemplateRequest; private boolean includeDefaults = false; - public Request() { - super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT); - } - - public Request(String templateName) { - super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT); - if (templateName == null) { - throw new IllegalArgumentException("template name cannot be null"); - } + public Request(TimeValue masterTimeout, String templateName) { + super(masterTimeout); this.templateName = templateName; } - public Request(TransportPutComposableIndexTemplateAction.Request indexTemplateRequest) { - super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT); - if (indexTemplateRequest == null) { - throw new IllegalArgumentException("index template body must be present"); - } - this.indexTemplateRequest = indexTemplateRequest; - } - + /** + * NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC we must remain able to read these requests until + * we no longer need to support calling this action remotely. + */ public Request(StreamInput in) throws IOException { super(in); templateName = in.readOptionalString(); @@ -73,16 +66,6 @@ public Request(StreamInput in) throws IOException { } } - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeOptionalString(templateName); - out.writeOptionalWriteable(indexTemplateRequest); - if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) { - out.writeBoolean(includeDefaults); - } - } - @Override public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; @@ -98,6 +81,11 @@ public ActionRequestValidationException validate() { return validationException; } + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + return new CancellableTask(id, type, action, "", parentTaskId, headers); + } + @Nullable public String getTemplateName() { return templateName; @@ -112,11 +100,6 @@ public TransportPutComposableIndexTemplateAction.Request getIndexTemplateRequest return indexTemplateRequest; } - public Request templateName(String templateName) { - this.templateName = templateName; - return this; - } - public Request indexTemplateRequest(TransportPutComposableIndexTemplateAction.Request indexTemplateRequest) { this.indexTemplateRequest = indexTemplateRequest; return this; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index d3d557b598b3a..74936128caa25 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -11,7 +11,8 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; +import org.elasticsearch.action.support.ChannelActionListener; +import org.elasticsearch.action.support.local.TransportLocalClusterStateAction; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; @@ -20,7 +21,6 @@ import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; @@ -31,6 +31,7 @@ import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.UpdateForV10; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.IndexSettingProvider; import org.elasticsearch.index.IndexSettingProviders; @@ -42,7 +43,6 @@ import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.tasks.Task; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -65,7 +65,7 @@ import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.resolveLifecycle; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.resolveSettings; -public class TransportSimulateIndexTemplateAction extends TransportMasterNodeReadAction< +public class TransportSimulateIndexTemplateAction extends TransportLocalClusterStateAction< SimulateIndexTemplateRequest, SimulateIndexTemplateResponse> { @@ -77,14 +77,18 @@ public class TransportSimulateIndexTemplateAction extends TransportMasterNodeRea private final ClusterSettings clusterSettings; private final boolean isDslOnlyMode; + /** + * NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC it must be registered with the TransportService until + * we no longer need to support calling this action remotely. + */ + @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT) + @SuppressWarnings("this-escape") @Inject public TransportSimulateIndexTemplateAction( TransportService transportService, ClusterService clusterService, - ThreadPool threadPool, MetadataIndexTemplateService indexTemplateService, ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver, NamedXContentRegistry xContentRegistry, IndicesService indicesService, SystemIndices systemIndices, @@ -92,13 +96,9 @@ public TransportSimulateIndexTemplateAction( ) { super( SimulateIndexTemplateAction.NAME, - transportService, - clusterService, - threadPool, actionFilters, - SimulateIndexTemplateRequest::new, - indexNameExpressionResolver, - SimulateIndexTemplateResponse::new, + transportService.getTaskManager(), + clusterService, EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.indexTemplateService = indexTemplateService; @@ -108,10 +108,19 @@ public TransportSimulateIndexTemplateAction( this.indexSettingProviders = indexSettingProviders.getIndexSettingProviders(); this.clusterSettings = clusterService.getClusterSettings(); this.isDslOnlyMode = isDataStreamsLifecycleOnlyMode(clusterService.getSettings()); + + transportService.registerRequestHandler( + actionName, + executor, + false, + true, + SimulateIndexTemplateRequest::new, + (request, channel, task) -> executeDirect(task, request, new ChannelActionListener<>(channel)) + ); } @Override - protected void masterOperation( + protected void localClusterStateOperation( Task task, SimulateIndexTemplateRequest request, ClusterState state, diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java index 30bbad0b57df0..692f027b23f9e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java @@ -11,26 +11,26 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; +import org.elasticsearch.action.support.ChannelActionListener; +import org.elasticsearch.action.support.local.TransportLocalClusterStateAction; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.UpdateForV10; import org.elasticsearch.index.IndexSettingProvider; import org.elasticsearch.index.IndexSettingProviders; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.tasks.Task; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -48,7 +48,7 @@ * Handles simulating an index template either by name (looking it up in the * cluster state), or by a provided template configuration */ -public class TransportSimulateTemplateAction extends TransportMasterNodeReadAction< +public class TransportSimulateTemplateAction extends TransportLocalClusterStateAction< SimulateTemplateAction.Request, SimulateIndexTemplateResponse> { @@ -60,14 +60,18 @@ public class TransportSimulateTemplateAction extends TransportMasterNodeReadActi private final ClusterSettings clusterSettings; private final boolean isDslOnlyMode; + /** + * NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC it must be registered with the TransportService until + * we no longer need to support calling this action remotely. + */ + @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT) + @SuppressWarnings("this-escape") @Inject public TransportSimulateTemplateAction( TransportService transportService, ClusterService clusterService, - ThreadPool threadPool, MetadataIndexTemplateService indexTemplateService, ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver, NamedXContentRegistry xContentRegistry, IndicesService indicesService, SystemIndices systemIndices, @@ -75,13 +79,9 @@ public TransportSimulateTemplateAction( ) { super( SimulateTemplateAction.NAME, - transportService, - clusterService, - threadPool, actionFilters, - SimulateTemplateAction.Request::new, - indexNameExpressionResolver, - SimulateIndexTemplateResponse::new, + transportService.getTaskManager(), + clusterService, EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.indexTemplateService = indexTemplateService; @@ -91,10 +91,19 @@ public TransportSimulateTemplateAction( this.indexSettingProviders = indexSettingProviders.getIndexSettingProviders(); this.clusterSettings = clusterService.getClusterSettings(); this.isDslOnlyMode = isDataStreamsLifecycleOnlyMode(clusterService.getSettings()); + + transportService.registerRequestHandler( + actionName, + executor, + false, + true, + SimulateTemplateAction.Request::new, + (request, channel, task) -> executeDirect(task, request, new ChannelActionListener<>(channel)) + ); } @Override - protected void masterOperation( + protected void localClusterStateOperation( Task task, SimulateTemplateAction.Request request, ClusterState state, diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateIndexTemplateAction.java index 4cc010c15ffb9..1689d234387cc 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateIndexTemplateAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestCancellableNodeClient; import org.elasticsearch.rest.action.RestToXContentListener; import java.io.IOException; @@ -41,8 +42,10 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - SimulateIndexTemplateRequest simulateIndexTemplateRequest = new SimulateIndexTemplateRequest(request.param("name")); - simulateIndexTemplateRequest.masterNodeTimeout(getMasterNodeTimeout(request)); + SimulateIndexTemplateRequest simulateIndexTemplateRequest = new SimulateIndexTemplateRequest( + getMasterNodeTimeout(request), + request.param("name") + ); simulateIndexTemplateRequest.includeDefaults(request.paramAsBoolean("include_defaults", false)); if (request.hasContent()) { TransportPutComposableIndexTemplateAction.Request indexTemplateRequest = new TransportPutComposableIndexTemplateAction.Request( @@ -57,7 +60,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC simulateIndexTemplateRequest.indexTemplateRequest(indexTemplateRequest); } - return channel -> client.execute( + return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()).execute( SimulateIndexTemplateAction.INSTANCE, simulateIndexTemplateRequest, new RestToXContentListener<>(channel) diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateTemplateAction.java index cc2e3136a4163..47b68e24a9ed0 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateTemplateAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestCancellableNodeClient; import org.elasticsearch.rest.action.RestToXContentListener; import java.io.IOException; @@ -39,8 +40,10 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { - SimulateTemplateAction.Request simulateRequest = new SimulateTemplateAction.Request(); - simulateRequest.templateName(request.param("name")); + SimulateTemplateAction.Request simulateRequest = new SimulateTemplateAction.Request( + getMasterNodeTimeout(request), + request.param("name") + ); simulateRequest.includeDefaults(request.paramAsBoolean("include_defaults", false)); if (request.hasContent()) { TransportPutComposableIndexTemplateAction.Request indexTemplateRequest = new TransportPutComposableIndexTemplateAction.Request( @@ -54,8 +57,11 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli simulateRequest.indexTemplateRequest(indexTemplateRequest); } - simulateRequest.masterNodeTimeout(getMasterNodeTimeout(request)); - return channel -> client.execute(SimulateTemplateAction.INSTANCE, simulateRequest, new RestToXContentListener<>(channel)); + return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()).execute( + SimulateTemplateAction.INSTANCE, + simulateRequest, + new RestToXContentListener<>(channel) + ); } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java index 02a4c134ccd15..1d861b340543d 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java @@ -12,45 +12,21 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; -import org.elasticsearch.cluster.metadata.ComposableIndexTemplateTests; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Template; -import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.test.ESTestCase; import java.util.List; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; -public class SimulateIndexTemplateRequestTests extends AbstractWireSerializingTestCase { - - @Override - protected Writeable.Reader instanceReader() { - return SimulateIndexTemplateRequest::new; - } - - @Override - protected SimulateIndexTemplateRequest createTestInstance() { - SimulateIndexTemplateRequest req = new SimulateIndexTemplateRequest(randomAlphaOfLength(10)); - TransportPutComposableIndexTemplateAction.Request newTemplateRequest = new TransportPutComposableIndexTemplateAction.Request( - randomAlphaOfLength(4) - ); - newTemplateRequest.indexTemplate(ComposableIndexTemplateTests.randomInstance()); - req.indexTemplateRequest(newTemplateRequest); - req.includeDefaults(randomBoolean()); - return req; - } - - @Override - protected SimulateIndexTemplateRequest mutateInstance(SimulateIndexTemplateRequest instance) { - return randomValueOtherThan(instance, this::createTestInstance); - } +public class SimulateIndexTemplateRequestTests extends ESTestCase { public void testIndexNameCannotBeNullOrEmpty() { - expectThrows(IllegalArgumentException.class, () -> new SimulateIndexTemplateRequest((String) null)); - expectThrows(IllegalArgumentException.class, () -> new SimulateIndexTemplateRequest("")); + expectThrows(IllegalArgumentException.class, () -> new SimulateIndexTemplateRequest(TEST_REQUEST_TIMEOUT, null)); + expectThrows(IllegalArgumentException.class, () -> new SimulateIndexTemplateRequest(TEST_REQUEST_TIMEOUT, "")); } public void testAddingGlobalTemplateWithHiddenIndexSettingIsIllegal() { @@ -60,7 +36,7 @@ public void testAddingGlobalTemplateWithHiddenIndexSettingIsIllegal() { TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request("test"); request.indexTemplate(globalTemplate); - SimulateIndexTemplateRequest simulateRequest = new SimulateIndexTemplateRequest("testing"); + SimulateIndexTemplateRequest simulateRequest = new SimulateIndexTemplateRequest(TEST_REQUEST_TIMEOUT, "testing"); simulateRequest.indexTemplateRequest(request); ActionRequestValidationException validationException = simulateRequest.validate(); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateRequestTests.java index 14ebf260d3bf9..6163566a5e593 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateRequestTests.java @@ -12,49 +12,17 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; -import org.elasticsearch.cluster.metadata.ComposableIndexTemplateTests; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Template; -import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.test.ESTestCase; import java.util.List; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; -public class SimulateTemplateRequestTests extends AbstractWireSerializingTestCase { - - @Override - protected Writeable.Reader instanceReader() { - return SimulateTemplateAction.Request::new; - } - - @Override - protected SimulateTemplateAction.Request createTestInstance() { - SimulateTemplateAction.Request req = new SimulateTemplateAction.Request(randomAlphaOfLength(10)); - TransportPutComposableIndexTemplateAction.Request newTemplateRequest = new TransportPutComposableIndexTemplateAction.Request( - randomAlphaOfLength(4) - ); - newTemplateRequest.indexTemplate(ComposableIndexTemplateTests.randomInstance()); - req.indexTemplateRequest(newTemplateRequest); - req.includeDefaults(randomBoolean()); - return req; - } - - @Override - protected SimulateTemplateAction.Request mutateInstance(SimulateTemplateAction.Request instance) { - return randomValueOtherThan(instance, this::createTestInstance); - } - - public void testIndexNameCannotBeNullOrEmpty() { - expectThrows(IllegalArgumentException.class, () -> new SimulateTemplateAction.Request((String) null)); - expectThrows( - IllegalArgumentException.class, - () -> new SimulateTemplateAction.Request((TransportPutComposableIndexTemplateAction.Request) null) - ); - } +public class SimulateTemplateRequestTests extends ESTestCase { public void testAddingGlobalTemplateWithHiddenIndexSettingIsIllegal() { Template template = new Template(Settings.builder().put(IndexMetadata.SETTING_INDEX_HIDDEN, true).build(), null, null); @@ -63,7 +31,7 @@ public void testAddingGlobalTemplateWithHiddenIndexSettingIsIllegal() { TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request("test"); request.indexTemplate(globalTemplate); - SimulateTemplateAction.Request simulateRequest = new SimulateTemplateAction.Request("testing"); + SimulateTemplateAction.Request simulateRequest = new SimulateTemplateAction.Request(TEST_REQUEST_TIMEOUT, "testing"); simulateRequest.indexTemplateRequest(request); ActionRequestValidationException validationException = simulateRequest.validate(); From 4ccd377d27f34f8e9b1c0618e10083af0460b5d0 Mon Sep 17 00:00:00 2001 From: Niels Bauman <33722607+nielsbauman@users.noreply.github.com> Date: Tue, 14 Jan 2025 13:45:58 +1000 Subject: [PATCH 07/31] Run `TransportClusterGetSettingsAction` on local node (#119831) This action solely needs the cluster state, it can run on any node. Additionally, it needs to be cancellable to avoid doing unnecessary work after a client failure or timeout. The `?local` parameter becomes a no-op and is marked as deprecated. --- docs/changelog/119831.yaml | 5 ++ .../http/RestActionCancellationIT.java | 5 ++ .../api/cluster.get_settings.json | 2 +- .../settings/ClusterGetSettingsAction.java | 36 ++++++++------ .../TransportClusterGetSettingsAction.java | 41 ++++++++++------ .../cluster/RestClusterGetSettingsAction.java | 12 ++--- .../ClusterGetSettingsSerializationTests.java | 49 ------------------- .../settings/ClusterGetSettingsTests.java | 10 ++-- 8 files changed, 66 insertions(+), 94 deletions(-) create mode 100644 docs/changelog/119831.yaml delete mode 100644 server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsSerializationTests.java diff --git a/docs/changelog/119831.yaml b/docs/changelog/119831.yaml new file mode 100644 index 0000000000000..61c09d7d54de0 --- /dev/null +++ b/docs/changelog/119831.yaml @@ -0,0 +1,5 @@ +pr: 119831 +summary: Run `TransportClusterGetSettingsAction` on local node +area: Infra/Settings +type: enhancement +issues: [] diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestActionCancellationIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestActionCancellationIT.java index 0af12a41fa979..c48ae9ba1843b 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestActionCancellationIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestActionCancellationIT.java @@ -12,6 +12,7 @@ import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.elasticsearch.action.admin.cluster.health.TransportClusterHealthAction; +import org.elasticsearch.action.admin.cluster.settings.ClusterGetSettingsAction; import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction; import org.elasticsearch.action.admin.indices.recovery.RecoveryAction; @@ -98,6 +99,10 @@ public void testSimulateIndexTemplateCancellation() { ); } + public void testClusterGetSettingsCancellation() { + runRestActionCancellationTest(new Request(HttpGet.METHOD_NAME, "/_cluster/settings"), ClusterGetSettingsAction.NAME); + } + private void runRestActionCancellationTest(Request request, String actionName) { final var node = usually() ? internalCluster().getRandomNodeName() : internalCluster().startCoordinatingOnlyNode(Settings.EMPTY); diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_settings.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_settings.json index 5862804257c67..5004ab8de697d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_settings.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_settings.json @@ -26,7 +26,7 @@ }, "master_timeout":{ "type":"time", - "description":"Explicit operation timeout for connection to master node" + "description":"Timeout for waiting for new cluster state in case it is blocked" }, "timeout":{ "type":"time", diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsAction.java index 7e3c38c735091..ca02d19749ae7 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsAction.java @@ -13,13 +13,18 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.support.master.MasterNodeReadRequest; +import org.elasticsearch.action.support.local.LocalClusterStateRequest; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.core.UpdateForV10; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; import java.io.IOException; +import java.util.Map; import java.util.Objects; public class ClusterGetSettingsAction extends ActionType { @@ -34,25 +39,29 @@ public ClusterGetSettingsAction() { /** * Request to retrieve the cluster settings */ - public static class Request extends MasterNodeReadRequest { + public static class Request extends LocalClusterStateRequest { public Request(TimeValue masterNodeTimeout) { super(masterNodeTimeout); } + /** + * NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC we must remain able to read these requests until + * we no longer need to support calling this action remotely. + */ + @UpdateForV10(owner = UpdateForV10.Owner.CORE_INFRA) public Request(StreamInput in) throws IOException { super(in); assert in.getTransportVersion().onOrAfter(TransportVersions.V_8_3_0); } @Override - public void writeTo(StreamOutput out) throws IOException { - assert out.getTransportVersion().onOrAfter(TransportVersions.V_8_3_0); - super.writeTo(out); + public ActionRequestValidationException validate() { + return null; } @Override - public ActionRequestValidationException validate() { - return null; + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + return new CancellableTask(id, type, action, "", parentTaskId, headers); } } @@ -79,20 +88,17 @@ public int hashCode() { return Objects.hash(persistentSettings, transientSettings, settings); } - public Response(StreamInput in) throws IOException { - super(in); - assert in.getTransportVersion().onOrAfter(TransportVersions.V_8_3_0); - persistentSettings = Settings.readSettingsFromStream(in); - transientSettings = Settings.readSettingsFromStream(in); - settings = Settings.readSettingsFromStream(in); - } - public Response(Settings persistentSettings, Settings transientSettings, Settings settings) { this.persistentSettings = Objects.requireNonNullElse(persistentSettings, Settings.EMPTY); this.transientSettings = Objects.requireNonNullElse(transientSettings, Settings.EMPTY); this.settings = Objects.requireNonNullElse(settings, Settings.EMPTY); } + /** + * NB prior to 9.0 get-component was a TransportMasterNodeReadAction so for BwC we must remain able to write these responses until + * we no longer need to support calling this action remotely. + */ + @UpdateForV10(owner = UpdateForV10.Owner.CORE_INFRA) @Override public void writeTo(StreamOutput out) throws IOException { assert out.getTransportVersion().onOrAfter(TransportVersions.V_8_3_0); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterGetSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterGetSettingsAction.java index dce6a38001392..71b976e012aad 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterGetSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterGetSettingsAction.java @@ -11,57 +11,66 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; +import org.elasticsearch.action.support.ChannelActionListener; +import org.elasticsearch.action.support.local.TransportLocalClusterStateAction; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.UpdateForV10; import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -public class TransportClusterGetSettingsAction extends TransportMasterNodeReadAction< +public class TransportClusterGetSettingsAction extends TransportLocalClusterStateAction< ClusterGetSettingsAction.Request, ClusterGetSettingsAction.Response> { private final SettingsFilter settingsFilter; + /** + * NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC it must be registered with the TransportService until + * we no longer need to support calling this action remotely. + */ + @UpdateForV10(owner = UpdateForV10.Owner.CORE_INFRA) + @SuppressWarnings("this-escape") @Inject public TransportClusterGetSettingsAction( TransportService transportService, ClusterService clusterService, - ThreadPool threadPool, SettingsFilter settingsFilter, - ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver + ActionFilters actionFilters ) { super( ClusterGetSettingsAction.NAME, - false, - transportService, - clusterService, - threadPool, actionFilters, - ClusterGetSettingsAction.Request::new, - indexNameExpressionResolver, - ClusterGetSettingsAction.Response::new, + transportService.getTaskManager(), + clusterService, EsExecutors.DIRECT_EXECUTOR_SERVICE ); - this.settingsFilter = settingsFilter; + + transportService.registerRequestHandler( + actionName, + executor, + false, + true, + ClusterGetSettingsAction.Request::new, + (request, channel, task) -> executeDirect(task, request, new ChannelActionListener<>(channel)) + ); } @Override - protected void masterOperation( + protected void localClusterStateOperation( Task task, ClusterGetSettingsAction.Request request, ClusterState state, ActionListener listener ) throws Exception { + ((CancellableTask) task).ensureNotCancelled(); Metadata metadata = state.metadata(); listener.onResponse( new ClusterGetSettingsAction.Response( diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterGetSettingsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterGetSettingsAction.java index ca9e4abcaeec7..477cc1acb8319 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterGetSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterGetSettingsAction.java @@ -11,15 +11,16 @@ import org.elasticsearch.action.admin.cluster.settings.ClusterGetSettingsAction; import org.elasticsearch.action.admin.cluster.settings.RestClusterGetSettingsResponse; -import org.elasticsearch.action.support.master.MasterNodeReadRequest; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestUtils; import org.elasticsearch.rest.Scope; import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestCancellableNodeClient; import org.elasticsearch.rest.action.RestToXContentListener; import java.io.IOException; @@ -52,19 +53,14 @@ public String getName() { return "cluster_get_settings_action"; } - private static void setUpRequestParams(MasterNodeReadRequest clusterRequest, RestRequest request) { - clusterRequest.local(request.paramAsBoolean("local", clusterRequest.local())); - } - @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { final boolean renderDefaults = request.paramAsBoolean("include_defaults", false); ClusterGetSettingsAction.Request clusterSettingsRequest = new ClusterGetSettingsAction.Request(getMasterNodeTimeout(request)); + RestUtils.consumeDeprecatedLocalParameter(request); - setUpRequestParams(clusterSettingsRequest, request); - - return channel -> client.execute( + return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()).execute( ClusterGetSettingsAction.INSTANCE, clusterSettingsRequest, new RestToXContentListener(channel).map( diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsSerializationTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsSerializationTests.java deleted file mode 100644 index 5954de0586985..0000000000000 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsSerializationTests.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.action.admin.cluster.settings; - -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.test.AbstractWireSerializingTestCase; - -public class ClusterGetSettingsSerializationTests extends AbstractWireSerializingTestCase { - @Override - protected Writeable.Reader instanceReader() { - return ClusterGetSettingsAction.Response::new; - } - - @Override - protected ClusterGetSettingsAction.Response createTestInstance() { - final Settings persistentSettings = Settings.builder() - .put("persistent.foo.filtered", "bar") - .put("persistent.foo.non_filtered", "baz") - .build(); - - final Settings transientSettings = Settings.builder() - .put("transient.foo.filtered", "bar") - .put("transient.foo.non_filtered", "baz") - .build(); - - final Settings allSettings = Settings.builder().put(persistentSettings).put(transientSettings).build(); - - return new ClusterGetSettingsAction.Response(persistentSettings, transientSettings, allSettings); - } - - @Override - protected ClusterGetSettingsAction.Response mutateInstance(ClusterGetSettingsAction.Response instance) { - final Settings otherSettings = Settings.builder().put("random.setting", randomAlphaOfLength(randomIntBetween(1, 12))).build(); - return switch (between(0, 2)) { - case 0 -> new ClusterGetSettingsAction.Response(otherSettings, instance.transientSettings(), instance.settings()); - case 1 -> new ClusterGetSettingsAction.Response(instance.persistentSettings(), otherSettings, instance.settings()); - case 2 -> new ClusterGetSettingsAction.Response(instance.persistentSettings(), instance.transientSettings(), otherSettings); - default -> throw new IllegalStateException("Unexpected switch value"); - }; - } -} diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsTests.java index 47f3f71bcc11b..58341b7d7fb9a 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsTests.java @@ -12,17 +12,18 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import java.util.List; +import java.util.Map; import static org.mockito.Mockito.mock; @@ -54,10 +55,8 @@ public void testTransportFilters() throws Exception { TransportClusterGetSettingsAction action = new TransportClusterGetSettingsAction( transportService, mock(ClusterService.class), - threadPool, filter, - mock(ActionFilters.class), - mock(IndexNameExpressionResolver.class) + mock(ActionFilters.class) ); final Settings persistentSettings = Settings.builder() @@ -74,7 +73,8 @@ public void testTransportFilters() throws Exception { final ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE).metadata(metadata).build(); final PlainActionFuture future = new PlainActionFuture<>(); - action.masterOperation(null, null, clusterState, future); + final var task = new CancellableTask(1, "test", ClusterGetSettingsAction.NAME, "", null, Map.of()); + action.localClusterStateOperation(task, null, clusterState, future); assertTrue(future.isDone()); final ClusterGetSettingsAction.Response response = future.get(); From 476797e523ff7c226c7cb850066684b1ec4a6e25 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Tue, 14 Jan 2025 16:29:16 +1100 Subject: [PATCH 08/31] Mute org.elasticsearch.xpack.ilm.actions.SearchableSnapshotActionIT testUpdatePolicyToAddPhasesYieldsInvalidActionsToBeSkipped #118406 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 2ede31c946207..9c640909cee6a 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -265,6 +265,9 @@ tests: - class: org.elasticsearch.entitlement.qa.EntitlementsDeniedIT method: testCheckThrows {pathPrefix=denied_nonmodular actionName=sslSessionImpl_getSessionContext} issue: https://github.com/elastic/elasticsearch/issues/120054 +- class: org.elasticsearch.xpack.ilm.actions.SearchableSnapshotActionIT + method: testUpdatePolicyToAddPhasesYieldsInvalidActionsToBeSkipped + issue: https://github.com/elastic/elasticsearch/issues/118406 # Examples: # From b1f4fa7225c40192f2a8e4a538686d60e3002d64 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Tue, 14 Jan 2025 16:58:05 +1100 Subject: [PATCH 09/31] Mute org.elasticsearch.xpack.ml.integration.DatafeedJobsIT org.elasticsearch.xpack.ml.integration.DatafeedJobsIT #120088 --- muted-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 9c640909cee6a..64087660a9b64 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -268,6 +268,8 @@ tests: - class: org.elasticsearch.xpack.ilm.actions.SearchableSnapshotActionIT method: testUpdatePolicyToAddPhasesYieldsInvalidActionsToBeSkipped issue: https://github.com/elastic/elasticsearch/issues/118406 +- class: org.elasticsearch.xpack.ml.integration.DatafeedJobsIT + issue: https://github.com/elastic/elasticsearch/issues/120088 # Examples: # From 4b52cf4f2f51c33369a4aefb58dd8a1cce841e77 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Tue, 14 Jan 2025 08:24:58 +0100 Subject: [PATCH 10/31] Fix potential file leak in ES816BinaryQuantizedVectorsWriter (#120014) We are creating tmp files that might not get closed if an exception happens just after it. This commit makes sure all errors are handle properly and files are getting closed and deleted. --- docs/changelog/120014.yaml | 6 ++ muted-tests.yml | 3 - .../ES816BinaryQuantizedVectorsWriter.java | 81 +++++++++++-------- 3 files changed, 55 insertions(+), 35 deletions(-) create mode 100644 docs/changelog/120014.yaml diff --git a/docs/changelog/120014.yaml b/docs/changelog/120014.yaml new file mode 100644 index 0000000000000..bef1f3ba49939 --- /dev/null +++ b/docs/changelog/120014.yaml @@ -0,0 +1,6 @@ +pr: 120014 +summary: Fix potential file leak in ES816BinaryQuantizedVectorsWriter +area: Search +type: bug +issues: + - 119981 diff --git a/muted-tests.yml b/muted-tests.yml index 64087660a9b64..3e69b7be0b2c3 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -242,9 +242,6 @@ tests: - class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT method: testMountSearchableSnapshot {p0=[9.0.0, 9.0.0, 8.18.0]} issue: https://github.com/elastic/elasticsearch/issues/119980 -- class: org.elasticsearch.index.codec.vectors.es816.ES816HnswBinaryQuantizedVectorsFormatTests - method: testRandomExceptions - issue: https://github.com/elastic/elasticsearch/issues/119981 - class: org.elasticsearch.multi_cluster.MultiClusterYamlTestSuiteIT issue: https://github.com/elastic/elasticsearch/issues/119983 - class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsWriter.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsWriter.java index 4d97235c5fae5..61bd5323b5b43 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsWriter.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsWriter.java @@ -437,32 +437,35 @@ private CloseableRandomVectorScorerSupplier mergeOneFieldToIndex( float[] centroid, float cDotC ) throws IOException { - long vectorDataOffset = binarizedVectorData.alignFilePointer(Float.BYTES); - final IndexOutput tempQuantizedVectorData = segmentWriteState.directory.createTempOutput( - binarizedVectorData.getName(), - "temp", - segmentWriteState.context - ); - final IndexOutput tempScoreQuantizedVectorData = segmentWriteState.directory.createTempOutput( - binarizedVectorData.getName(), - "score_temp", - segmentWriteState.context - ); - IndexInput binarizedDataInput = null; - IndexInput binarizedScoreDataInput = null; - boolean success = false; - int descritizedDimension = BQVectorUtils.discretize(fieldInfo.getVectorDimension(), 64); - BinaryQuantizer quantizer = new BinaryQuantizer( + final long vectorDataOffset = binarizedVectorData.alignFilePointer(Float.BYTES); + final int descritizedDimension = BQVectorUtils.discretize(fieldInfo.getVectorDimension(), 64); + final BinaryQuantizer quantizer = new BinaryQuantizer( fieldInfo.getVectorDimension(), descritizedDimension, fieldInfo.getVectorSimilarityFunction() ); + + IndexOutput tempQuantizedVectorData = null; + IndexOutput tempScoreQuantizedVectorData = null; + final DocsWithFieldSet docsWithField; + boolean success = false; + try { + tempQuantizedVectorData = segmentWriteState.directory.createTempOutput( + binarizedVectorData.getName(), + "temp", + segmentWriteState.context + ); + tempScoreQuantizedVectorData = segmentWriteState.directory.createTempOutput( + binarizedVectorData.getName(), + "score_temp", + segmentWriteState.context + ); FloatVectorValues floatVectorValues = KnnVectorsWriter.MergedVectorValues.mergeFloatVectorValues(fieldInfo, mergeState); if (fieldInfo.getVectorSimilarityFunction() == COSINE) { floatVectorValues = new NormalizedFloatVectorValues(floatVectorValues); } - DocsWithFieldSet docsWithField = writeBinarizedVectorAndQueryData( + docsWithField = writeBinarizedVectorAndQueryData( tempQuantizedVectorData, tempScoreQuantizedVectorData, floatVectorValues, @@ -470,13 +473,30 @@ private CloseableRandomVectorScorerSupplier mergeOneFieldToIndex( quantizer ); CodecUtil.writeFooter(tempQuantizedVectorData); - IOUtils.close(tempQuantizedVectorData); + CodecUtil.writeFooter(tempScoreQuantizedVectorData); + success = true; + } finally { + if (success) { + IOUtils.close(tempQuantizedVectorData, tempScoreQuantizedVectorData); + } else { + IOUtils.closeWhileHandlingException(tempQuantizedVectorData, tempScoreQuantizedVectorData); + if (tempQuantizedVectorData != null) { + IOUtils.deleteFilesIgnoringExceptions(segmentWriteState.directory, tempQuantizedVectorData.getName()); + } + if (tempScoreQuantizedVectorData != null) { + IOUtils.deleteFilesIgnoringExceptions(segmentWriteState.directory, tempScoreQuantizedVectorData.getName()); + } + } + } + + IndexInput binarizedDataInput = null; + IndexInput binarizedScoreDataInput = null; + success = false; + try { binarizedDataInput = segmentWriteState.directory.openInput(tempQuantizedVectorData.getName(), segmentWriteState.context); binarizedVectorData.copyBytes(binarizedDataInput, binarizedDataInput.length() - CodecUtil.footerLength()); - long vectorDataLength = binarizedVectorData.getFilePointer() - vectorDataOffset; + final long vectorDataLength = binarizedVectorData.getFilePointer() - vectorDataOffset; CodecUtil.retrieveChecksum(binarizedDataInput); - CodecUtil.writeFooter(tempScoreQuantizedVectorData); - IOUtils.close(tempScoreQuantizedVectorData); binarizedScoreDataInput = segmentWriteState.directory.openInput( tempScoreQuantizedVectorData.getName(), segmentWriteState.context @@ -490,10 +510,9 @@ private CloseableRandomVectorScorerSupplier mergeOneFieldToIndex( cDotC, docsWithField ); - success = true; final IndexInput finalBinarizedDataInput = binarizedDataInput; final IndexInput finalBinarizedScoreDataInput = binarizedScoreDataInput; - OffHeapBinarizedVectorValues vectorValues = new OffHeapBinarizedVectorValues.DenseOffHeapVectorValues( + final OffHeapBinarizedVectorValues vectorValues = new OffHeapBinarizedVectorValues.DenseOffHeapVectorValues( fieldInfo.getVectorDimension(), docsWithField.cardinality(), centroid, @@ -503,7 +522,7 @@ private CloseableRandomVectorScorerSupplier mergeOneFieldToIndex( vectorsScorer, finalBinarizedDataInput ); - RandomVectorScorerSupplier scorerSupplier = vectorsScorer.getRandomVectorScorerSupplier( + final RandomVectorScorerSupplier scorerSupplier = vectorsScorer.getRandomVectorScorerSupplier( fieldInfo.getVectorSimilarityFunction(), new OffHeapBinarizedQueryVectorValues( finalBinarizedScoreDataInput, @@ -513,22 +532,20 @@ private CloseableRandomVectorScorerSupplier mergeOneFieldToIndex( ), vectorValues ); + final String tempQuantizedVectorDataName = tempQuantizedVectorData.getName(); + final String tempScoreQuantizedVectorDataName = tempScoreQuantizedVectorData.getName(); + success = true; return new BinarizedCloseableRandomVectorScorerSupplier(scorerSupplier, vectorValues, () -> { IOUtils.close(finalBinarizedDataInput, finalBinarizedScoreDataInput); IOUtils.deleteFilesIgnoringExceptions( segmentWriteState.directory, - tempQuantizedVectorData.getName(), - tempScoreQuantizedVectorData.getName() + tempQuantizedVectorDataName, + tempScoreQuantizedVectorDataName ); }); } finally { if (success == false) { - IOUtils.closeWhileHandlingException( - tempQuantizedVectorData, - tempScoreQuantizedVectorData, - binarizedDataInput, - binarizedScoreDataInput - ); + IOUtils.closeWhileHandlingException(binarizedDataInput, binarizedScoreDataInput); IOUtils.deleteFilesIgnoringExceptions( segmentWriteState.directory, tempQuantizedVectorData.getName(), From 5123b948a9c320ee806b2f7016e394bbbfb1a518 Mon Sep 17 00:00:00 2001 From: Moritz Mack Date: Tue, 14 Jan 2025 08:41:11 +0100 Subject: [PATCH 11/31] Remove deprecated tracing.apm.* settings for v9 (#119926) --- .../server/cli/APMJvmOptions.java | 50 +----- .../server/cli/APMJvmOptionsTests.java | 143 ++++++------------ docs/changelog/119926.yaml | 11 ++ .../org/elasticsearch/telemetry/apm/APM.java | 9 +- .../apm/internal/APMAgentSettings.java | 96 ++---------- .../apm/internal/APMAgentSettingsTests.java | 116 +------------- 6 files changed, 81 insertions(+), 344 deletions(-) create mode 100644 docs/changelog/119926.yaml diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java index c3b9768946767..1e57d9fab7cfd 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java @@ -187,20 +187,12 @@ static String agentCommandLineOption(Path agentJar, Path tmpPropertiesFile) { static void extractSecureSettings(SecureSettings secrets, Map propertiesMap) { final Set settingNames = secrets.getSettingNames(); for (String key : List.of("api_key", "secret_token")) { - for (String prefix : List.of("telemetry.", "tracing.apm.")) { - if (settingNames.contains(prefix + key)) { - if (propertiesMap.containsKey(key)) { - throw new IllegalStateException( - Strings.format("Duplicate telemetry setting: [telemetry.%s] and [tracing.apm.%s]", key, key) - ); - } - - try (SecureString token = secrets.getString(prefix + key)) { - propertiesMap.put(key, token.toString()); - } + String prefix = "telemetry."; + if (settingNames.contains(prefix + key)) { + try (SecureString token = secrets.getString(prefix + key)) { + propertiesMap.put(key, token.toString()); } } - } } @@ -227,44 +219,12 @@ private static Map extractDynamicSettings(Map pr static Map extractApmSettings(Settings settings) throws UserException { final Map propertiesMap = new HashMap<>(); - // tracing.apm.agent. is deprecated by telemetry.agent. final String telemetryAgentPrefix = "telemetry.agent."; - final String deprecatedTelemetryAgentPrefix = "tracing.apm.agent."; final Settings telemetryAgentSettings = settings.getByPrefix(telemetryAgentPrefix); telemetryAgentSettings.keySet().forEach(key -> propertiesMap.put(key, String.valueOf(telemetryAgentSettings.get(key)))); - final Settings apmAgentSettings = settings.getByPrefix(deprecatedTelemetryAgentPrefix); - for (String key : apmAgentSettings.keySet()) { - if (propertiesMap.containsKey(key)) { - throw new IllegalStateException( - Strings.format( - "Duplicate telemetry setting: [%s%s] and [%s%s]", - telemetryAgentPrefix, - key, - deprecatedTelemetryAgentPrefix, - key - ) - ); - } - propertiesMap.put(key, String.valueOf(apmAgentSettings.get(key))); - } - StringJoiner globalLabels = extractGlobalLabels(telemetryAgentPrefix, propertiesMap, settings); - if (globalLabels.length() == 0) { - globalLabels = extractGlobalLabels(deprecatedTelemetryAgentPrefix, propertiesMap, settings); - } else { - StringJoiner tracingGlobalLabels = extractGlobalLabels(deprecatedTelemetryAgentPrefix, propertiesMap, settings); - if (tracingGlobalLabels.length() != 0) { - throw new IllegalArgumentException( - "Cannot have global labels with tracing.agent prefix [" - + globalLabels - + "] and telemetry.apm.agent prefix [" - + tracingGlobalLabels - + "]" - ); - } - } if (globalLabels.length() > 0) { propertiesMap.put("global_labels", globalLabels.toString()); } @@ -274,7 +234,7 @@ static Map extractApmSettings(Settings settings) throws UserExce if (propertiesMap.containsKey(key)) { throw new UserException( ExitCodes.CONFIG, - "Do not set a value for [tracing.apm.agent." + key + "], as this is configured automatically by Elasticsearch" + "Do not set a value for [telemetry.agent." + key + "], as this is configured automatically by Elasticsearch" ); } } diff --git a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/APMJvmOptionsTests.java b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/APMJvmOptionsTests.java index a7ba8eb11fbcc..0e067afc1aa73 100644 --- a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/APMJvmOptionsTests.java +++ b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/APMJvmOptionsTests.java @@ -25,18 +25,15 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.function.Function; import static org.elasticsearch.test.MapMatcher.matchesMap; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; @@ -82,109 +79,63 @@ public void testFileDeleteWorks() throws IOException { } public void testExtractSecureSettings() { - MockSecureSettings duplicateSecureSettings = new MockSecureSettings(); + MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("telemetry.secret_token", "token"); + secureSettings.setString("telemetry.api_key", "key"); - for (String prefix : List.of("telemetry.", "tracing.apm.")) { - MockSecureSettings secureSettings = new MockSecureSettings(); - secureSettings.setString(prefix + "secret_token", "token"); - secureSettings.setString(prefix + "api_key", "key"); - - duplicateSecureSettings.setString(prefix + "api_key", "secret"); - - Map propertiesMap = new HashMap<>(); - APMJvmOptions.extractSecureSettings(secureSettings, propertiesMap); - - assertThat(propertiesMap, matchesMap(Map.of("secret_token", "token", "api_key", "key"))); - } - - Exception exception = expectThrows( - IllegalStateException.class, - () -> APMJvmOptions.extractSecureSettings(duplicateSecureSettings, new HashMap<>()) - ); - assertThat(exception.getMessage(), containsString("Duplicate telemetry setting")); - assertThat(exception.getMessage(), containsString("telemetry.api_key")); - assertThat(exception.getMessage(), containsString("tracing.apm.api_key")); + Map propertiesMap = new HashMap<>(); + APMJvmOptions.extractSecureSettings(secureSettings, propertiesMap); + assertThat(propertiesMap, matchesMap(Map.of("secret_token", "token", "api_key", "key"))); } public void testExtractSettings() throws UserException { - Function buildSettings = (prefix) -> Settings.builder() - .put(prefix + "server_url", "https://myurl:443") - .put(prefix + "service_node_name", "instance-0000000001"); - - for (String prefix : List.of("tracing.apm.agent.", "telemetry.agent.")) { - var name = "APM Tracing"; - var deploy = "123"; - var org = "456"; - var extracted = APMJvmOptions.extractApmSettings( - buildSettings.apply(prefix) - .put(prefix + "global_labels.deployment_name", name) - .put(prefix + "global_labels.deployment_id", deploy) - .put(prefix + "global_labels.organization_id", org) - .build() - ); - - assertThat( - extracted, - allOf( - hasEntry("server_url", "https://myurl:443"), - hasEntry("service_node_name", "instance-0000000001"), - hasEntry(equalTo("global_labels"), not(endsWith(","))), // test that we have collapsed all global labels into one - not(hasKey("global_labels.organization_id")) // tests that we strip out the top level label keys - ) - ); - - List labels = Arrays.stream(extracted.get("global_labels").split(",")).toList(); - assertThat(labels, hasSize(3)); - assertThat(labels, containsInAnyOrder("deployment_name=APM Tracing", "organization_id=" + org, "deployment_id=" + deploy)); - - // test replacing with underscores and skipping empty - name = "APM=Tracing"; - deploy = ""; - org = ",456"; - extracted = APMJvmOptions.extractApmSettings( - buildSettings.apply(prefix) - .put(prefix + "global_labels.deployment_name", name) - .put(prefix + "global_labels.deployment_id", deploy) - .put(prefix + "global_labels.organization_id", org) - .build() - ); - labels = Arrays.stream(extracted.get("global_labels").split(",")).toList(); - assertThat(labels, hasSize(2)); - assertThat(labels, containsInAnyOrder("deployment_name=APM_Tracing", "organization_id=_456")); - } - - IllegalStateException err = expectThrows( - IllegalStateException.class, - () -> APMJvmOptions.extractApmSettings( - Settings.builder() - .put("tracing.apm.agent.server_url", "https://myurl:443") - .put("telemetry.agent.server_url", "https://myurl-2:443") - .build() - ) - ); - assertThat(err.getMessage(), is("Duplicate telemetry setting: [telemetry.agent.server_url] and [tracing.apm.agent.server_url]")); - } - - public void testNoMixedLabels() { - String telemetryAgent = "telemetry.agent."; - String tracingAgent = "tracing.apm.agent."; - Settings settings = Settings.builder() - .put("tracing.apm.enabled", true) - .put(telemetryAgent + "server_url", "https://myurl:443") - .put(telemetryAgent + "service_node_name", "instance-0000000001") - .put(tracingAgent + "global_labels.deployment_id", "123") - .put(telemetryAgent + "global_labels.organization_id", "456") + Settings defaults = Settings.builder() + .put("telemetry.agent.server_url", "https://myurl:443") + .put("telemetry.agent.service_node_name", "instance-0000000001") .build(); - IllegalArgumentException err = assertThrows(IllegalArgumentException.class, () -> APMJvmOptions.extractApmSettings(settings)); + var name = "APM Tracing"; + var deploy = "123"; + var org = "456"; + var extracted = APMJvmOptions.extractApmSettings( + Settings.builder() + .put(defaults) + .put("telemetry.agent.global_labels.deployment_name", name) + .put("telemetry.agent.global_labels.deployment_id", deploy) + .put("telemetry.agent.global_labels.organization_id", org) + .build() + ); + assertThat( - err.getMessage(), - is( - "Cannot have global labels with tracing.agent prefix [organization_id=456] and" - + " telemetry.apm.agent prefix [deployment_id=123]" + extracted, + allOf( + hasEntry("server_url", "https://myurl:443"), + hasEntry("service_node_name", "instance-0000000001"), + hasEntry(equalTo("global_labels"), not(endsWith(","))), // test that we have collapsed all global labels into one + not(hasKey("global_labels.organization_id")) // tests that we strip out the top level label keys ) ); + + List labels = Arrays.stream(extracted.get("global_labels").split(",")).toList(); + assertThat(labels, hasSize(3)); + assertThat(labels, containsInAnyOrder("deployment_name=APM Tracing", "organization_id=" + org, "deployment_id=" + deploy)); + + // test replacing with underscores and skipping empty + name = "APM=Tracing"; + deploy = ""; + org = ",456"; + extracted = APMJvmOptions.extractApmSettings( + Settings.builder() + .put(defaults) + .put("telemetry.agent.global_labels.deployment_name", name) + .put("telemetry.agent.global_labels.deployment_id", deploy) + .put("telemetry.agent.global_labels.organization_id", org) + .build() + ); + labels = Arrays.stream(extracted.get("global_labels").split(",")).toList(); + assertThat(labels, hasSize(2)); + assertThat(labels, containsInAnyOrder("deployment_name=APM_Tracing", "organization_id=_456")); } private Path makeFakeAgentJar() throws IOException { diff --git a/docs/changelog/119926.yaml b/docs/changelog/119926.yaml new file mode 100644 index 0000000000000..3afafd5b2117f --- /dev/null +++ b/docs/changelog/119926.yaml @@ -0,0 +1,11 @@ +pr: 119926 +summary: "Deprecated tracing.apm.* settings got removed." +area: Infra/Metrics +type: breaking +issues: [] +breaking: + title: "Deprecated tracing.apm.* settings got removed." + area: Cluster and node setting + details: Deprecated `tracing.apm.*` settings got removed, use respective `telemetry.*` / `telemetry.tracing.*` settings instead. + impact: 9.x nodes will refuse to start if any such setting (including secret settings) is still present. + notable: false diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/APM.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/APM.java index 339a4ec24ca13..43447cfa21a62 100644 --- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/APM.java +++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/APM.java @@ -92,14 +92,7 @@ public List> getSettings() { APMAgentSettings.TELEMETRY_TRACING_ENABLED_SETTING, APMAgentSettings.TELEMETRY_TRACING_NAMES_INCLUDE_SETTING, APMAgentSettings.TELEMETRY_TRACING_NAMES_EXCLUDE_SETTING, - APMAgentSettings.TELEMETRY_TRACING_SANITIZE_FIELD_NAMES, - // The settings below are deprecated and are currently kept as fallback. - APMAgentSettings.TRACING_APM_SECRET_TOKEN_SETTING, - APMAgentSettings.TRACING_APM_API_KEY_SETTING, - APMAgentSettings.TRACING_APM_ENABLED_SETTING, - APMAgentSettings.TRACING_APM_NAMES_INCLUDE_SETTING, - APMAgentSettings.TRACING_APM_NAMES_EXCLUDE_SETTING, - APMAgentSettings.TRACING_APM_SANITIZE_FIELD_NAMES + APMAgentSettings.TELEMETRY_TRACING_SANITIZE_FIELD_NAMES ); } } diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettings.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettings.java index f66683a787bc0..8647761e2defe 100644 --- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettings.java +++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettings.java @@ -25,9 +25,7 @@ import java.util.List; import java.util.Objects; import java.util.Set; -import java.util.function.Function; -import static org.elasticsearch.common.settings.Setting.Property.Deprecated; import static org.elasticsearch.common.settings.Setting.Property.NodeScope; import static org.elasticsearch.common.settings.Setting.Property.OperatorDynamic; @@ -101,9 +99,6 @@ public void setAgentSetting(String key, String value) { private static final String TELEMETRY_SETTING_PREFIX = "telemetry."; - // The old legacy prefix - private static final String LEGACY_TRACING_APM_SETTING_PREFIX = "tracing.apm."; - /** * Allow-list of APM agent config keys users are permitted to configure. * @see APM Java Agent Configuration @@ -248,56 +243,24 @@ private static Setting concreteAgentSetting(String namespace, String qua public static final Setting.AffixSetting APM_AGENT_SETTINGS = Setting.prefixKeySetting( TELEMETRY_SETTING_PREFIX + "agent.", - LEGACY_TRACING_APM_SETTING_PREFIX + "agent.", - (namespace, qualifiedKey) -> qualifiedKey.startsWith(LEGACY_TRACING_APM_SETTING_PREFIX) - ? concreteAgentSetting(namespace, qualifiedKey, NodeScope, OperatorDynamic, Deprecated) - : concreteAgentSetting(namespace, qualifiedKey, NodeScope, OperatorDynamic) + null, // no fallback + (namespace, qualifiedKey) -> concreteAgentSetting(namespace, qualifiedKey, NodeScope, OperatorDynamic) ); - /** - * @deprecated in favor of TELEMETRY_TRACING_NAMES_INCLUDE_SETTING. - */ - @Deprecated - public static final Setting> TRACING_APM_NAMES_INCLUDE_SETTING = Setting.stringListSetting( - LEGACY_TRACING_APM_SETTING_PREFIX + "names.include", - OperatorDynamic, - NodeScope, - Deprecated - ); - - public static final Setting> TELEMETRY_TRACING_NAMES_INCLUDE_SETTING = Setting.listSetting( + public static final Setting> TELEMETRY_TRACING_NAMES_INCLUDE_SETTING = Setting.stringListSetting( TELEMETRY_SETTING_PREFIX + "tracing.names.include", - TRACING_APM_NAMES_INCLUDE_SETTING, - Function.identity(), OperatorDynamic, NodeScope ); - /** - * @deprecated in favor of TELEMETRY_TRACING_NAMES_EXCLUDE_SETTING. - */ - @Deprecated - public static final Setting> TRACING_APM_NAMES_EXCLUDE_SETTING = Setting.stringListSetting( - LEGACY_TRACING_APM_SETTING_PREFIX + "names.exclude", - OperatorDynamic, - NodeScope, - Deprecated - ); - - public static final Setting> TELEMETRY_TRACING_NAMES_EXCLUDE_SETTING = Setting.listSetting( + public static final Setting> TELEMETRY_TRACING_NAMES_EXCLUDE_SETTING = Setting.stringListSetting( TELEMETRY_SETTING_PREFIX + "tracing.names.exclude", - TRACING_APM_NAMES_EXCLUDE_SETTING, - Function.identity(), OperatorDynamic, NodeScope ); - /** - * @deprecated in favor of TELEMETRY_TRACING_SANITIZE_FIELD_NAMES. - */ - @Deprecated - public static final Setting> TRACING_APM_SANITIZE_FIELD_NAMES = Setting.stringListSetting( - LEGACY_TRACING_APM_SETTING_PREFIX + "sanitize_field_names", + public static final Setting> TELEMETRY_TRACING_SANITIZE_FIELD_NAMES = Setting.stringListSetting( + TELEMETRY_SETTING_PREFIX + "tracing.sanitize_field_names", List.of( "password", "passwd", @@ -313,33 +276,12 @@ private static Setting concreteAgentSetting(String namespace, String qua "set-cookie" ), OperatorDynamic, - NodeScope, - Deprecated - ); - - public static final Setting> TELEMETRY_TRACING_SANITIZE_FIELD_NAMES = Setting.listSetting( - TELEMETRY_SETTING_PREFIX + "tracing.sanitize_field_names", - TRACING_APM_SANITIZE_FIELD_NAMES, - Function.identity(), - OperatorDynamic, NodeScope ); - /** - * @deprecated in favor of TELEMETRY_TRACING_ENABLED_SETTING. - */ - @Deprecated - public static final Setting TRACING_APM_ENABLED_SETTING = Setting.boolSetting( - LEGACY_TRACING_APM_SETTING_PREFIX + "enabled", - false, - OperatorDynamic, - NodeScope, - Deprecated - ); - public static final Setting TELEMETRY_TRACING_ENABLED_SETTING = Setting.boolSetting( TELEMETRY_SETTING_PREFIX + "tracing.enabled", - TRACING_APM_ENABLED_SETTING, + false, OperatorDynamic, NodeScope ); @@ -351,33 +293,13 @@ private static Setting concreteAgentSetting(String namespace, String qua NodeScope ); - /** - * @deprecated in favor of TELEMETRY_SECRET_TOKEN_SETTING. - */ - @Deprecated - public static final Setting TRACING_APM_SECRET_TOKEN_SETTING = SecureSetting.secureString( - LEGACY_TRACING_APM_SETTING_PREFIX + "secret_token", - null, - Deprecated - ); - public static final Setting TELEMETRY_SECRET_TOKEN_SETTING = SecureSetting.secureString( TELEMETRY_SETTING_PREFIX + "secret_token", - TRACING_APM_SECRET_TOKEN_SETTING - ); - - /** - * @deprecated in favor of TELEMETRY_API_KEY_SETTING. - */ - @Deprecated - public static final Setting TRACING_APM_API_KEY_SETTING = SecureSetting.secureString( - LEGACY_TRACING_APM_SETTING_PREFIX + "api_key", - null, - Deprecated + null ); public static final Setting TELEMETRY_API_KEY_SETTING = SecureSetting.secureString( TELEMETRY_SETTING_PREFIX + "api_key", - TRACING_APM_API_KEY_SETTING + null ); } diff --git a/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettingsTests.java b/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettingsTests.java index a60048c82a3c9..5516672420924 100644 --- a/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettingsTests.java +++ b/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettingsTests.java @@ -11,8 +11,6 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.ClusterSettings; -import org.elasticsearch.common.settings.MockSecureSettings; -import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; import org.mockito.Mockito; @@ -21,21 +19,13 @@ import java.util.Set; import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.APM_AGENT_SETTINGS; -import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TELEMETRY_API_KEY_SETTING; import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TELEMETRY_METRICS_ENABLED_SETTING; -import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TELEMETRY_SECRET_TOKEN_SETTING; import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TELEMETRY_TRACING_ENABLED_SETTING; import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TELEMETRY_TRACING_NAMES_EXCLUDE_SETTING; import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TELEMETRY_TRACING_NAMES_INCLUDE_SETTING; import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TELEMETRY_TRACING_SANITIZE_FIELD_NAMES; -import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TRACING_APM_API_KEY_SETTING; -import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TRACING_APM_ENABLED_SETTING; -import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TRACING_APM_NAMES_EXCLUDE_SETTING; -import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TRACING_APM_NAMES_INCLUDE_SETTING; -import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TRACING_APM_SANITIZE_FIELD_NAMES; -import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TRACING_APM_SECRET_TOKEN_SETTING; -import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.hasItem; import static org.mockito.Mockito.clearInvocations; import static org.mockito.Mockito.mock; @@ -70,14 +60,6 @@ public void testEnableTracing() { } } - public void testEnableTracingUsingLegacySetting() { - Settings settings = Settings.builder().put(TRACING_APM_ENABLED_SETTING.getKey(), true).build(); - apmAgentSettings.initAgentSystemProperties(settings); - - verify(apmAgentSettings).setAgentSetting("recording", "true"); - assertWarnings("[tracing.apm.enabled] setting was deprecated in Elasticsearch and will be removed in a future release."); - } - public void testEnableMetrics() { for (boolean tracingEnabled : List.of(true, false)) { clearInvocations(apmAgentSettings, apmTelemetryProvider.getMeterService()); @@ -121,14 +103,6 @@ public void testDisableTracing() { } } - public void testDisableTracingUsingLegacySetting() { - Settings settings = Settings.builder().put(TRACING_APM_ENABLED_SETTING.getKey(), false).build(); - apmAgentSettings.initAgentSystemProperties(settings); - - verify(apmAgentSettings).setAgentSetting("recording", "false"); - assertWarnings("[tracing.apm.enabled] setting was deprecated in Elasticsearch and will be removed in a future release."); - } - public void testDisableMetrics() { for (boolean tracingEnabled : List.of(true, false)) { clearInvocations(apmAgentSettings, apmTelemetryProvider.getMeterService()); @@ -181,70 +155,18 @@ public void testSetAgentSettings() { verify(apmAgentSettings).setAgentSetting("span_compression_enabled", "true"); } - public void testSetAgentsSettingsWithLegacyPrefix() { - Settings settings = Settings.builder() - .put(TELEMETRY_TRACING_ENABLED_SETTING.getKey(), true) - .put("tracing.apm.agent.span_compression_enabled", "true") - .build(); - apmAgentSettings.initAgentSystemProperties(settings); - - verify(apmAgentSettings).setAgentSetting("recording", "true"); - verify(apmAgentSettings).setAgentSetting("span_compression_enabled", "true"); - assertWarnings( - "[tracing.apm.agent.span_compression_enabled] setting was deprecated in Elasticsearch and will be removed in a future release." - ); - } - /** * Check that invalid or forbidden APM agent settings are rejected. */ public void testRejectForbiddenOrUnknownAgentSettings() { - List prefixes = List.of(APM_AGENT_SETTINGS.getKey(), "tracing.apm.agent."); - for (String prefix : prefixes) { - Settings settings = Settings.builder().put(prefix + "unknown", "true").build(); - Exception exception = expectThrows(IllegalArgumentException.class, () -> APM_AGENT_SETTINGS.getAsMap(settings)); - assertThat(exception.getMessage(), containsString("[" + prefix + "unknown]")); - } - // though, accept / ignore nested global_labels - for (String prefix : prefixes) { - Settings settings = Settings.builder().put(prefix + "global_labels.abc", "123").build(); - APMAgentSettings.APM_AGENT_SETTINGS.getAsMap(settings); - - if (prefix.startsWith("tracing.apm.agent.")) { - assertWarnings( - "[tracing.apm.agent.global_labels.abc] setting was deprecated in Elasticsearch and will be removed in a future release." - ); - } - } - } - - public void testTelemetryTracingNamesIncludeFallback() { - Settings settings = Settings.builder().put(TRACING_APM_NAMES_INCLUDE_SETTING.getKey(), "abc,xyz").build(); - - List included = TELEMETRY_TRACING_NAMES_INCLUDE_SETTING.get(settings); - - assertThat(included, containsInAnyOrder("abc", "xyz")); - assertWarnings("[tracing.apm.names.include] setting was deprecated in Elasticsearch and will be removed in a future release."); - } - - public void testTelemetryTracingNamesExcludeFallback() { - Settings settings = Settings.builder().put(TRACING_APM_NAMES_EXCLUDE_SETTING.getKey(), "abc,xyz").build(); - - List included = TELEMETRY_TRACING_NAMES_EXCLUDE_SETTING.get(settings); - - assertThat(included, containsInAnyOrder("abc", "xyz")); - assertWarnings("[tracing.apm.names.exclude] setting was deprecated in Elasticsearch and will be removed in a future release."); - } - - public void testTelemetryTracingSanitizeFieldNamesFallback() { - Settings settings = Settings.builder().put(TRACING_APM_SANITIZE_FIELD_NAMES.getKey(), "abc,xyz").build(); - - List included = TELEMETRY_TRACING_SANITIZE_FIELD_NAMES.get(settings); + String prefix = APM_AGENT_SETTINGS.getKey(); + Settings settings = Settings.builder().put(prefix + "unknown", "true").build(); + Exception exception = expectThrows(IllegalArgumentException.class, () -> APM_AGENT_SETTINGS.getAsMap(settings)); + assertThat(exception.getMessage(), containsString("[" + prefix + "unknown]")); - assertThat(included, containsInAnyOrder("abc", "xyz")); - assertWarnings( - "[tracing.apm.sanitize_field_names] setting was deprecated in Elasticsearch and will be removed in a future release." - ); + // though, accept / ignore nested global_labels + var map = APMAgentSettings.APM_AGENT_SETTINGS.getAsMap(Settings.builder().put(prefix + "global_labels.abc", "123").build()); + assertThat(map, hasEntry("global_labels.abc", "123")); } public void testTelemetryTracingSanitizeFieldNamesFallbackDefault() { @@ -252,28 +174,6 @@ public void testTelemetryTracingSanitizeFieldNamesFallbackDefault() { assertThat(included, hasItem("password")); // and more defaults } - public void testTelemetrySecretTokenFallback() { - MockSecureSettings secureSettings = new MockSecureSettings(); - secureSettings.setString(TRACING_APM_SECRET_TOKEN_SETTING.getKey(), "verysecret"); - Settings settings = Settings.builder().setSecureSettings(secureSettings).build(); - - try (SecureString secureString = TELEMETRY_SECRET_TOKEN_SETTING.get(settings)) { - assertEquals("verysecret", secureString.toString()); - } - assertWarnings("[tracing.apm.secret_token] setting was deprecated in Elasticsearch and will be removed in a future release."); - } - - public void testTelemetryApiKeyFallback() { - MockSecureSettings secureSettings = new MockSecureSettings(); - secureSettings.setString(TRACING_APM_API_KEY_SETTING.getKey(), "abc"); - Settings settings = Settings.builder().setSecureSettings(secureSettings).build(); - - try (SecureString secureString = TELEMETRY_API_KEY_SETTING.get(settings)) { - assertEquals("abc", secureString.toString()); - } - assertWarnings("[tracing.apm.api_key] setting was deprecated in Elasticsearch and will be removed in a future release."); - } - /** * Check that invalid or forbidden APM agent settings are rejected if their last part resembles an allowed setting. */ From 1be9253779797157c397e8a1931d06fe613ca35b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenzo=20Dematt=C3=A9?= Date: Tue, 14 Jan 2025 09:08:15 +0100 Subject: [PATCH 12/31] [Entitlements] Network entitlement classes + Datagram socket check functions (#119735) --- .../bridge/EntitlementChecker.java | 30 ++++ .../qa/common/DummyImplementations.java | 78 ++++++++- .../common/RestEntitlementsCheckAction.java | 160 ++++++++++-------- .../plugin-metadata/entitlement-policy.yaml | 5 + .../plugin-metadata/entitlement-policy.yaml | 5 + .../api/ElasticsearchEntitlementChecker.java | 71 ++++++++ .../runtime/policy/NetworkEntitlement.java | 107 ++++++++++++ .../runtime/policy/PolicyManager.java | 34 +++- .../runtime/policy/PolicyParser.java | 3 +- .../policy/NetworkEntitlementTests.java | 49 ++++++ .../runtime/policy/PolicyParserTests.java | 16 ++ 11 files changed, 485 insertions(+), 73 deletions(-) create mode 100644 libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/NetworkEntitlement.java create mode 100644 libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/NetworkEntitlementTests.java diff --git a/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java b/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java index 3a359eb921fc8..69fc57973f68a 100644 --- a/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java +++ b/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java @@ -13,10 +13,16 @@ import java.io.PrintStream; import java.io.PrintWriter; import java.net.ContentHandlerFactory; +import java.net.DatagramPacket; +import java.net.DatagramSocket; import java.net.DatagramSocketImplFactory; import java.net.FileNameMap; +import java.net.InetAddress; +import java.net.MulticastSocket; +import java.net.NetworkInterface; import java.net.ProxySelector; import java.net.ResponseCache; +import java.net.SocketAddress; import java.net.SocketImplFactory; import java.net.URL; import java.net.URLStreamHandler; @@ -189,4 +195,28 @@ public interface EntitlementChecker { // The only implementation of SSLSession#getSessionContext(); unfortunately it's an interface, so we need to check the implementation void check$sun_security_ssl_SSLSessionImpl$getSessionContext(Class callerClass, SSLSession sslSession); + + void check$java_net_DatagramSocket$bind(Class callerClass, DatagramSocket that, SocketAddress addr); + + void check$java_net_DatagramSocket$connect(Class callerClass, DatagramSocket that, InetAddress addr); + + void check$java_net_DatagramSocket$connect(Class callerClass, DatagramSocket that, SocketAddress addr); + + void check$java_net_DatagramSocket$send(Class callerClass, DatagramSocket that, DatagramPacket p); + + void check$java_net_DatagramSocket$receive(Class callerClass, DatagramSocket that, DatagramPacket p); + + void check$java_net_DatagramSocket$joinGroup(Class callerClass, DatagramSocket that, SocketAddress addr, NetworkInterface ni); + + void check$java_net_DatagramSocket$leaveGroup(Class callerClass, DatagramSocket that, SocketAddress addr, NetworkInterface ni); + + void check$java_net_MulticastSocket$joinGroup(Class callerClass, MulticastSocket that, InetAddress addr); + + void check$java_net_MulticastSocket$joinGroup(Class callerClass, MulticastSocket that, SocketAddress addr, NetworkInterface ni); + + void check$java_net_MulticastSocket$leaveGroup(Class callerClass, MulticastSocket that, InetAddress addr); + + void check$java_net_MulticastSocket$leaveGroup(Class callerClass, MulticastSocket that, SocketAddress addr, NetworkInterface ni); + + void check$java_net_MulticastSocket$send(Class callerClass, MulticastSocket that, DatagramPacket p, byte ttl); } diff --git a/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/DummyImplementations.java b/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/DummyImplementations.java index 6dbb684c71514..fae873123528d 100644 --- a/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/DummyImplementations.java +++ b/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/DummyImplementations.java @@ -9,8 +9,15 @@ package org.elasticsearch.entitlement.qa.common; +import java.io.IOException; +import java.net.DatagramPacket; +import java.net.DatagramSocket; +import java.net.DatagramSocketImpl; import java.net.InetAddress; +import java.net.NetworkInterface; import java.net.Socket; +import java.net.SocketAddress; +import java.net.SocketException; import java.security.cert.Certificate; import java.text.BreakIterator; import java.text.Collator; @@ -327,8 +334,77 @@ public Socket createSocket(Socket s, String host, int port, boolean autoClose) { } } + static class DummyDatagramSocket extends DatagramSocket { + DummyDatagramSocket() throws SocketException { + super(new DatagramSocketImpl() { + @Override + protected void create() throws SocketException {} + + @Override + protected void bind(int lport, InetAddress laddr) throws SocketException {} + + @Override + protected void send(DatagramPacket p) throws IOException {} + + @Override + protected int peek(InetAddress i) throws IOException { + return 0; + } + + @Override + protected int peekData(DatagramPacket p) throws IOException { + return 0; + } + + @Override + protected void receive(DatagramPacket p) throws IOException {} + + @Override + protected void setTTL(byte ttl) throws IOException {} + + @Override + protected byte getTTL() throws IOException { + return 0; + } + + @Override + protected void setTimeToLive(int ttl) throws IOException {} + + @Override + protected int getTimeToLive() throws IOException { + return 0; + } + + @Override + protected void join(InetAddress inetaddr) throws IOException {} + + @Override + protected void leave(InetAddress inetaddr) throws IOException {} + + @Override + protected void joinGroup(SocketAddress mcastaddr, NetworkInterface netIf) throws IOException {} + + @Override + protected void leaveGroup(SocketAddress mcastaddr, NetworkInterface netIf) throws IOException {} + + @Override + protected void close() {} + + @Override + public void setOption(int optID, Object value) throws SocketException {} + + @Override + public Object getOption(int optID) throws SocketException { + return null; + } + + @Override + protected void connect(InetAddress address, int port) throws SocketException {} + }); + } + } + private static RuntimeException unexpected() { return new IllegalStateException("This method isn't supposed to be called"); } - } diff --git a/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/RestEntitlementsCheckAction.java b/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/RestEntitlementsCheckAction.java index 1dd8daf556226..3a5480f468528 100644 --- a/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/RestEntitlementsCheckAction.java +++ b/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/RestEntitlementsCheckAction.java @@ -11,6 +11,7 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.Strings; +import org.elasticsearch.core.CheckedRunnable; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.entitlement.qa.common.DummyImplementations.DummyBreakIteratorProvider; import org.elasticsearch.entitlement.qa.common.DummyImplementations.DummyCalendarDataProvider; @@ -32,14 +33,18 @@ import org.elasticsearch.rest.RestStatus; import java.io.IOException; -import java.io.UncheckedIOException; +import java.net.DatagramPacket; import java.net.DatagramSocket; import java.net.HttpURLConnection; +import java.net.InetAddress; +import java.net.InetSocketAddress; import java.net.MalformedURLException; +import java.net.NetworkInterface; import java.net.ProxySelector; import java.net.ResponseCache; import java.net.ServerSocket; import java.net.Socket; +import java.net.SocketException; import java.net.URL; import java.net.URLClassLoader; import java.net.URLConnection; @@ -71,20 +76,20 @@ public class RestEntitlementsCheckAction extends BaseRestHandler { public static final Thread NO_OP_SHUTDOWN_HOOK = new Thread(() -> {}, "Shutdown hook for testing"); private final String prefix; - record CheckAction(Runnable action, boolean isAlwaysDeniedToPlugins) { + record CheckAction(CheckedRunnable action, boolean isAlwaysDeniedToPlugins) { /** * These cannot be granted to plugins, so our test plugins cannot test the "allowed" case. * Used both for always-denied entitlements as well as those granted only to the server itself. */ - static CheckAction deniedToPlugins(Runnable action) { + static CheckAction deniedToPlugins(CheckedRunnable action) { return new CheckAction(action, true); } - static CheckAction forPlugins(Runnable action) { + static CheckAction forPlugins(CheckedRunnable action) { return new CheckAction(action, false); } - static CheckAction alwaysDenied(Runnable action) { + static CheckAction alwaysDenied(CheckedRunnable action) { return new CheckAction(action, true); } } @@ -142,7 +147,13 @@ static CheckAction alwaysDenied(Runnable action) { entry("createURLStreamHandlerProvider", alwaysDenied(RestEntitlementsCheckAction::createURLStreamHandlerProvider)), entry("createURLWithURLStreamHandler", alwaysDenied(RestEntitlementsCheckAction::createURLWithURLStreamHandler)), entry("createURLWithURLStreamHandler2", alwaysDenied(RestEntitlementsCheckAction::createURLWithURLStreamHandler2)), - entry("sslSessionImpl_getSessionContext", alwaysDenied(RestEntitlementsCheckAction::sslSessionImplGetSessionContext)) + entry("sslSessionImpl_getSessionContext", alwaysDenied(RestEntitlementsCheckAction::sslSessionImplGetSessionContext)), + entry("datagram_socket_bind", forPlugins(RestEntitlementsCheckAction::bindDatagramSocket)), + entry("datagram_socket_connect", forPlugins(RestEntitlementsCheckAction::connectDatagramSocket)), + entry("datagram_socket_send", forPlugins(RestEntitlementsCheckAction::sendDatagramSocket)), + entry("datagram_socket_receive", forPlugins(RestEntitlementsCheckAction::receiveDatagramSocket)), + entry("datagram_socket_join_group", forPlugins(RestEntitlementsCheckAction::joinGroupDatagramSocket)), + entry("datagram_socket_leave_group", forPlugins(RestEntitlementsCheckAction::leaveGroupDatagramSocket)) ); private static void createURLStreamHandlerProvider() { @@ -154,43 +165,33 @@ public URLStreamHandler createURLStreamHandler(String protocol) { }; } - private static void sslSessionImplGetSessionContext() { + private static void sslSessionImplGetSessionContext() throws IOException { SSLSocketFactory factory = HttpsURLConnection.getDefaultSSLSocketFactory(); try (SSLSocket socket = (SSLSocket) factory.createSocket()) { SSLSession session = socket.getSession(); session.getSessionContext(); - } catch (IOException e) { - throw new RuntimeException(e); } } @SuppressWarnings("deprecation") - private static void createURLWithURLStreamHandler() { - try { - var x = new URL("http", "host", 1234, "file", new URLStreamHandler() { - @Override - protected URLConnection openConnection(URL u) { - return null; - } - }); - } catch (MalformedURLException e) { - throw new RuntimeException(e); - } + private static void createURLWithURLStreamHandler() throws MalformedURLException { + var x = new URL("http", "host", 1234, "file", new URLStreamHandler() { + @Override + protected URLConnection openConnection(URL u) { + return null; + } + }); } @SuppressWarnings("deprecation") - private static void createURLWithURLStreamHandler2() { - try { - var x = new URL(null, "spec", new URLStreamHandler() { - @Override - protected URLConnection openConnection(URL u) { - return null; - } - }); - } catch (MalformedURLException e) { - throw new RuntimeException(e); - } + private static void createURLWithURLStreamHandler2() throws MalformedURLException { + var x = new URL(null, "spec", new URLStreamHandler() { + @Override + protected URLConnection openConnection(URL u) { + return null; + } + }); } private static void createInetAddressResolverProvider() { @@ -215,12 +216,8 @@ private static void setDefaultProxySelector() { ProxySelector.setDefault(null); } - private static void setDefaultSSLContext() { - try { - SSLContext.setDefault(SSLContext.getDefault()); - } catch (NoSuchAlgorithmException e) { - throw new RuntimeException(e); - } + private static void setDefaultSSLContext() throws NoSuchAlgorithmException { + SSLContext.setDefault(SSLContext.getDefault()); } private static void setDefaultHostnameVerifier() { @@ -246,28 +243,18 @@ private static void systemExit() { System.exit(123); } - private static void createClassLoader() { + private static void createClassLoader() throws IOException { try (var classLoader = new URLClassLoader("test", new URL[0], RestEntitlementsCheckAction.class.getClassLoader())) { logger.info("Created URLClassLoader [{}]", classLoader.getName()); - } catch (IOException e) { - throw new UncheckedIOException(e); } } - private static void processBuilder_start() { - try { - new ProcessBuilder("").start(); - } catch (IOException e) { - throw new IllegalStateException(e); - } + private static void processBuilder_start() throws IOException { + new ProcessBuilder("").start(); } - private static void processBuilder_startPipeline() { - try { - ProcessBuilder.startPipeline(List.of()); - } catch (IOException e) { - throw new IllegalStateException(e); - } + private static void processBuilder_startPipeline() throws IOException { + ProcessBuilder.startPipeline(List.of()); } private static void setHttpsConnectionProperties() { @@ -355,12 +342,8 @@ private static void setHttpsConnectionProperties() { @SuppressWarnings("deprecation") @SuppressForbidden(reason = "We're required to prevent calls to this forbidden API") - private static void datagramSocket$$setDatagramSocketImplFactory() { - try { - DatagramSocket.setDatagramSocketImplFactory(() -> { throw new IllegalStateException(); }); - } catch (IOException e) { - throw new IllegalStateException(e); - } + private static void datagramSocket$$setDatagramSocketImplFactory() throws IOException { + DatagramSocket.setDatagramSocketImplFactory(() -> { throw new IllegalStateException(); }); } private static void httpURLConnection$$setFollowRedirects() { @@ -369,22 +352,14 @@ private static void setHttpsConnectionProperties() { @SuppressWarnings("deprecation") @SuppressForbidden(reason = "We're required to prevent calls to this forbidden API") - private static void serverSocket$$setSocketFactory() { - try { - ServerSocket.setSocketFactory(() -> { throw new IllegalStateException(); }); - } catch (IOException e) { - throw new IllegalStateException(e); - } + private static void serverSocket$$setSocketFactory() throws IOException { + ServerSocket.setSocketFactory(() -> { throw new IllegalStateException(); }); } @SuppressWarnings("deprecation") @SuppressForbidden(reason = "We're required to prevent calls to this forbidden API") - private static void socket$$setSocketImplFactory() { - try { - Socket.setSocketImplFactory(() -> { throw new IllegalStateException(); }); - } catch (IOException e) { - throw new IllegalStateException(e); - } + private static void socket$$setSocketImplFactory() throws IOException { + Socket.setSocketImplFactory(() -> { throw new IllegalStateException(); }); } private static void url$$setURLStreamHandlerFactory() { @@ -399,6 +374,51 @@ private static void setHttpsConnectionProperties() { URLConnection.setContentHandlerFactory(__ -> { throw new IllegalStateException(); }); } + private static void bindDatagramSocket() throws SocketException { + try (var socket = new DatagramSocket(null)) { + socket.bind(null); + } + } + + @SuppressForbidden(reason = "testing entitlements") + private static void connectDatagramSocket() throws SocketException { + try (var socket = new DummyImplementations.DummyDatagramSocket()) { + socket.connect(new InetSocketAddress(1234)); + } + } + + private static void joinGroupDatagramSocket() throws IOException { + try (var socket = new DummyImplementations.DummyDatagramSocket()) { + socket.joinGroup( + new InetSocketAddress(InetAddress.getByAddress(new byte[] { (byte) 230, 0, 0, 1 }), 1234), + NetworkInterface.getByIndex(0) + ); + } + } + + private static void leaveGroupDatagramSocket() throws IOException { + try (var socket = new DummyImplementations.DummyDatagramSocket()) { + socket.leaveGroup( + new InetSocketAddress(InetAddress.getByAddress(new byte[] { (byte) 230, 0, 0, 1 }), 1234), + NetworkInterface.getByIndex(0) + ); + } + } + + @SuppressForbidden(reason = "testing entitlements") + private static void sendDatagramSocket() throws IOException { + try (var socket = new DummyImplementations.DummyDatagramSocket()) { + socket.send(new DatagramPacket(new byte[] { 0 }, 1, InetAddress.getLocalHost(), 1234)); + } + } + + @SuppressForbidden(reason = "testing entitlements") + private static void receiveDatagramSocket() throws IOException { + try (var socket = new DummyImplementations.DummyDatagramSocket()) { + socket.receive(new DatagramPacket(new byte[1], 1, InetAddress.getLocalHost(), 1234)); + } + } + public RestEntitlementsCheckAction(String prefix) { this.prefix = prefix; } diff --git a/libs/entitlement/qa/entitlement-allowed-nonmodular/src/main/plugin-metadata/entitlement-policy.yaml b/libs/entitlement/qa/entitlement-allowed-nonmodular/src/main/plugin-metadata/entitlement-policy.yaml index 30fc9f0abeec0..05a94f09264a8 100644 --- a/libs/entitlement/qa/entitlement-allowed-nonmodular/src/main/plugin-metadata/entitlement-policy.yaml +++ b/libs/entitlement/qa/entitlement-allowed-nonmodular/src/main/plugin-metadata/entitlement-policy.yaml @@ -1,3 +1,8 @@ ALL-UNNAMED: - create_class_loader - set_https_connection_properties + - network: + actions: + - listen + - accept + - connect diff --git a/libs/entitlement/qa/entitlement-allowed/src/main/plugin-metadata/entitlement-policy.yaml b/libs/entitlement/qa/entitlement-allowed/src/main/plugin-metadata/entitlement-policy.yaml index 0a25570a9f624..0d2c66c2daa2c 100644 --- a/libs/entitlement/qa/entitlement-allowed/src/main/plugin-metadata/entitlement-policy.yaml +++ b/libs/entitlement/qa/entitlement-allowed/src/main/plugin-metadata/entitlement-policy.yaml @@ -1,3 +1,8 @@ org.elasticsearch.entitlement.qa.common: - create_class_loader - set_https_connection_properties + - network: + actions: + - listen + - accept + - connect diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java index ca4aaceabcebf..dd39ec3c5fe43 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java @@ -10,16 +10,23 @@ package org.elasticsearch.entitlement.runtime.api; import org.elasticsearch.entitlement.bridge.EntitlementChecker; +import org.elasticsearch.entitlement.runtime.policy.NetworkEntitlement; import org.elasticsearch.entitlement.runtime.policy.PolicyManager; import java.io.InputStream; import java.io.PrintStream; import java.io.PrintWriter; import java.net.ContentHandlerFactory; +import java.net.DatagramPacket; +import java.net.DatagramSocket; import java.net.DatagramSocketImplFactory; import java.net.FileNameMap; +import java.net.InetAddress; +import java.net.MulticastSocket; +import java.net.NetworkInterface; import java.net.ProxySelector; import java.net.ResponseCache; +import java.net.SocketAddress; import java.net.SocketImplFactory; import java.net.URL; import java.net.URLStreamHandler; @@ -349,4 +356,68 @@ public ElasticsearchEntitlementChecker(PolicyManager policyManager) { public void check$sun_security_ssl_SSLSessionImpl$getSessionContext(Class callerClass, SSLSession sslSession) { policyManager.checkReadSensitiveNetworkInformation(callerClass); } + + @Override + public void check$java_net_DatagramSocket$bind(Class callerClass, DatagramSocket that, SocketAddress addr) { + policyManager.checkNetworkAccess(callerClass, NetworkEntitlement.LISTEN_ACTION); + } + + @Override + public void check$java_net_DatagramSocket$connect(Class callerClass, DatagramSocket that, InetAddress addr) { + policyManager.checkNetworkAccess(callerClass, NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION); + } + + @Override + public void check$java_net_DatagramSocket$connect(Class callerClass, DatagramSocket that, SocketAddress addr) { + policyManager.checkNetworkAccess(callerClass, NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION); + } + + @Override + public void check$java_net_DatagramSocket$send(Class callerClass, DatagramSocket that, DatagramPacket p) { + var actions = NetworkEntitlement.CONNECT_ACTION; + if (p.getAddress().isMulticastAddress()) { + actions |= NetworkEntitlement.ACCEPT_ACTION; + } + policyManager.checkNetworkAccess(callerClass, actions); + } + + @Override + public void check$java_net_DatagramSocket$receive(Class callerClass, DatagramSocket that, DatagramPacket p) { + policyManager.checkNetworkAccess(callerClass, NetworkEntitlement.ACCEPT_ACTION); + } + + @Override + public void check$java_net_DatagramSocket$joinGroup(Class caller, DatagramSocket that, SocketAddress addr, NetworkInterface ni) { + policyManager.checkNetworkAccess(caller, NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION); + } + + @Override + public void check$java_net_DatagramSocket$leaveGroup(Class caller, DatagramSocket that, SocketAddress addr, NetworkInterface ni) { + policyManager.checkNetworkAccess(caller, NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION); + } + + @Override + public void check$java_net_MulticastSocket$joinGroup(Class callerClass, MulticastSocket that, InetAddress addr) { + policyManager.checkNetworkAccess(callerClass, NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION); + } + + @Override + public void check$java_net_MulticastSocket$joinGroup(Class caller, MulticastSocket that, SocketAddress addr, NetworkInterface ni) { + policyManager.checkNetworkAccess(caller, NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION); + } + + @Override + public void check$java_net_MulticastSocket$leaveGroup(Class caller, MulticastSocket that, InetAddress addr) { + policyManager.checkNetworkAccess(caller, NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION); + } + + @Override + public void check$java_net_MulticastSocket$leaveGroup(Class caller, MulticastSocket that, SocketAddress addr, NetworkInterface ni) { + policyManager.checkNetworkAccess(caller, NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION); + } + + @Override + public void check$java_net_MulticastSocket$send(Class callerClass, MulticastSocket that, DatagramPacket p, byte ttl) { + policyManager.checkNetworkAccess(callerClass, NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION); + } } diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/NetworkEntitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/NetworkEntitlement.java new file mode 100644 index 0000000000000..b6c6a41d5be7f --- /dev/null +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/NetworkEntitlement.java @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy; + +import org.elasticsearch.core.Strings; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.StringJoiner; + +import static java.util.Map.entry; + +/** + * Describes a network entitlement (sockets) with actions. + */ +public class NetworkEntitlement implements Entitlement { + + public static final int LISTEN_ACTION = 0x1; + public static final int CONNECT_ACTION = 0x2; + public static final int ACCEPT_ACTION = 0x4; + + static final String LISTEN = "listen"; + static final String CONNECT = "connect"; + static final String ACCEPT = "accept"; + + private static final Map ACTION_MAP = Map.ofEntries( + entry(LISTEN, LISTEN_ACTION), + entry(CONNECT, CONNECT_ACTION), + entry(ACCEPT, ACCEPT_ACTION) + ); + + private final int actions; + + @ExternalEntitlement(parameterNames = { "actions" }, esModulesOnly = false) + public NetworkEntitlement(List actionsList) { + + int actionsInt = 0; + + for (String actionString : actionsList) { + var action = ACTION_MAP.get(actionString); + if (action == null) { + throw new IllegalArgumentException("unknown network action [" + actionString + "]"); + } + if ((actionsInt & action) == action) { + throw new IllegalArgumentException(Strings.format("network action [%s] specified multiple times", actionString)); + } + actionsInt |= action; + } + + this.actions = actionsInt; + } + + public static Object printActions(int actions) { + var joiner = new StringJoiner(","); + for (var entry : ACTION_MAP.entrySet()) { + var action = entry.getValue(); + if ((actions & action) == action) { + joiner.add(entry.getKey()); + } + } + return joiner.toString(); + } + + /** + * For the actions to match, the actions present in this entitlement must be a superset + * of the actions required by a check. + * There is only one "negative" case (action required by the check but not present in the entitlement), + * and it can be expressed efficiently via this truth table: + * this.actions | requiredActions | + * 0 | 0 | 0 + * 0 | 1 | 1 --> NOT this.action AND requiredActions + * 1 | 0 | 0 + * 1 | 1 | 0 + * + * @param requiredActions the actions required to be present for a check to pass + * @return true if requiredActions are present, false otherwise + */ + public boolean matchActions(int requiredActions) { + return (~this.actions & requiredActions) == 0; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + NetworkEntitlement that = (NetworkEntitlement) o; + return actions == that.actions; + } + + @Override + public int hashCode() { + return Objects.hash(actions); + } + + @Override + public String toString() { + return "NetworkEntitlement{actions=" + actions + '}'; + } +} diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java index 57449a23a8215..f039fbda3dfbd 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java @@ -52,7 +52,11 @@ public boolean hasEntitlement(Class entitlementClass) { } public Stream getEntitlements(Class entitlementClass) { - return entitlementsByType.get(entitlementClass).stream().map(entitlementClass::cast); + var entitlements = entitlementsByType.get(entitlementClass); + if (entitlements == null) { + return Stream.empty(); + } + return entitlements.stream().map(entitlementClass::cast); } } @@ -190,6 +194,34 @@ private String operationDescription(String methodName) { return methodName.substring(methodName.indexOf('$')); } + public void checkNetworkAccess(Class callerClass, int actions) { + var requestingClass = requestingClass(callerClass); + if (isTriviallyAllowed(requestingClass)) { + return; + } + + ModuleEntitlements entitlements = getEntitlements(requestingClass); + if (entitlements.getEntitlements(NetworkEntitlement.class).anyMatch(n -> n.matchActions(actions))) { + logger.debug( + () -> Strings.format( + "Entitled: class [%s], module [%s], entitlement [Network], actions [Ox%X]", + requestingClass, + requestingClass.getModule().getName(), + actions + ) + ); + return; + } + throw new NotEntitledException( + Strings.format( + "Missing entitlement: class [%s], module [%s], entitlement [Network], actions [%s]", + requestingClass, + requestingClass.getModule().getName(), + NetworkEntitlement.printActions(actions) + ) + ); + } + private void checkEntitlementPresent(Class callerClass, Class entitlementClass) { var requestingClass = requestingClass(callerClass); if (isTriviallyAllowed(requestingClass)) { diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java index 013acf8f22fae..ac4d4afdd97f8 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java @@ -37,7 +37,8 @@ public class PolicyParser { private static final Map> EXTERNAL_ENTITLEMENTS = Stream.of( FileEntitlement.class, CreateClassLoaderEntitlement.class, - SetHttpsConnectionPropertiesEntitlement.class + SetHttpsConnectionPropertiesEntitlement.class, + NetworkEntitlement.class ).collect(Collectors.toUnmodifiableMap(PolicyParser::getEntitlementTypeName, Function.identity())); protected final XContentParser policyParser; diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/NetworkEntitlementTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/NetworkEntitlementTests.java new file mode 100644 index 0000000000000..91051d48c365f --- /dev/null +++ b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/NetworkEntitlementTests.java @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy; + +import org.elasticsearch.test.ESTestCase; + +import java.util.List; + +import static org.hamcrest.Matchers.is; + +public class NetworkEntitlementTests extends ESTestCase { + + public void testMatchesActions() { + var listenEntitlement = new NetworkEntitlement(List.of(NetworkEntitlement.LISTEN)); + var emptyEntitlement = new NetworkEntitlement(List.of()); + var connectAcceptEntitlement = new NetworkEntitlement(List.of(NetworkEntitlement.CONNECT, NetworkEntitlement.ACCEPT)); + + assertThat(listenEntitlement.matchActions(0), is(true)); + assertThat(listenEntitlement.matchActions(NetworkEntitlement.LISTEN_ACTION), is(true)); + assertThat(listenEntitlement.matchActions(NetworkEntitlement.ACCEPT_ACTION), is(false)); + assertThat(listenEntitlement.matchActions(NetworkEntitlement.CONNECT_ACTION), is(false)); + assertThat(listenEntitlement.matchActions(NetworkEntitlement.LISTEN_ACTION | NetworkEntitlement.ACCEPT_ACTION), is(false)); + assertThat(listenEntitlement.matchActions(NetworkEntitlement.LISTEN_ACTION | NetworkEntitlement.CONNECT_ACTION), is(false)); + assertThat(listenEntitlement.matchActions(NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION), is(false)); + + assertThat(connectAcceptEntitlement.matchActions(0), is(true)); + assertThat(connectAcceptEntitlement.matchActions(NetworkEntitlement.LISTEN_ACTION), is(false)); + assertThat(connectAcceptEntitlement.matchActions(NetworkEntitlement.ACCEPT_ACTION), is(true)); + assertThat(connectAcceptEntitlement.matchActions(NetworkEntitlement.CONNECT_ACTION), is(true)); + assertThat(connectAcceptEntitlement.matchActions(NetworkEntitlement.LISTEN_ACTION | NetworkEntitlement.ACCEPT_ACTION), is(false)); + assertThat(connectAcceptEntitlement.matchActions(NetworkEntitlement.LISTEN_ACTION | NetworkEntitlement.CONNECT_ACTION), is(false)); + assertThat(connectAcceptEntitlement.matchActions(NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION), is(true)); + + assertThat(emptyEntitlement.matchActions(0), is(true)); + assertThat(emptyEntitlement.matchActions(NetworkEntitlement.LISTEN_ACTION), is(false)); + assertThat(emptyEntitlement.matchActions(NetworkEntitlement.ACCEPT_ACTION), is(false)); + assertThat(emptyEntitlement.matchActions(NetworkEntitlement.CONNECT_ACTION), is(false)); + assertThat(emptyEntitlement.matchActions(NetworkEntitlement.LISTEN_ACTION | NetworkEntitlement.ACCEPT_ACTION), is(false)); + assertThat(emptyEntitlement.matchActions(NetworkEntitlement.LISTEN_ACTION | NetworkEntitlement.CONNECT_ACTION), is(false)); + assertThat(emptyEntitlement.matchActions(NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION), is(false)); + } +} diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java index 4d17fc92e1578..1e0c31d2280b8 100644 --- a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java +++ b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java @@ -52,6 +52,22 @@ public void testPolicyBuilderOnExternalPlugin() throws IOException { assertEquals(expected, parsedPolicy); } + public void testParseNetwork() throws IOException { + Policy parsedPolicy = new PolicyParser(new ByteArrayInputStream(""" + entitlement-module-name: + - network: + actions: + - listen + - accept + - connect + """.getBytes(StandardCharsets.UTF_8)), "test-policy.yaml", false).parsePolicy(); + Policy expected = new Policy( + "test-policy.yaml", + List.of(new Scope("entitlement-module-name", List.of(new NetworkEntitlement(List.of("listen", "accept", "connect"))))) + ); + assertEquals(expected, parsedPolicy); + } + public void testParseCreateClassloader() throws IOException { Policy parsedPolicy = new PolicyParser(new ByteArrayInputStream(""" entitlement-module-name: From d953079380243f518ca3847fdcb4e1816a4c8163 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Tue, 14 Jan 2025 10:28:59 +0100 Subject: [PATCH 13/31] Cleanup swapSourceProvider(...) workaround (#118480) This reverts the workaround that was introduced in #117792 to avoid EOF error when an es|ql query uses multiple runtime fields that fallback to source when source mode is synthetic. This is now covered by the ReinitializingSourceProvider workaround that covers that and the concurrency problem. With this change, the main code for the required workarounds are now in isolated in ReinitializingSourceProvider. Additional another in `ReinitializingSourceProvider` was fixed, the issue was the lastSeenDoc field was reused overwritten by different threads, the latest commit moves the lastSeenDoc field to PerThreadSourceProvider so that each thread gets its own place to store the last seen docid. --- .../search/lookup/SearchLookup.java | 11 ---------- .../planner/EsPhysicalOperationProviders.java | 12 +---------- .../plugin/ReinitializingSourceProvider.java | 20 ++++++++++++------- 3 files changed, 14 insertions(+), 29 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java index 9eb0170af5efb..d899a390d8be7 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java @@ -102,14 +102,6 @@ private SearchLookup(SearchLookup searchLookup, Set fieldChain) { this.fieldLookupProvider = searchLookup.fieldLookupProvider; } - private SearchLookup(SearchLookup searchLookup, SourceProvider sourceProvider, Set fieldChain) { - this.fieldChain = Collections.unmodifiableSet(fieldChain); - this.sourceProvider = sourceProvider; - this.fieldTypeLookup = searchLookup.fieldTypeLookup; - this.fieldDataLookup = searchLookup.fieldDataLookup; - this.fieldLookupProvider = searchLookup.fieldLookupProvider; - } - /** * Creates a copy of the current {@link SearchLookup} that looks fields up in the same way, but also tracks field references * in order to detect cycles and prevent resolving fields that depend on more than {@link #MAX_FIELD_CHAIN_DEPTH} other fields. @@ -153,7 +145,4 @@ public Source getSource(LeafReaderContext ctx, int doc) throws IOException { return sourceProvider.getSource(ctx, doc); } - public SearchLookup swapSourceProvider(SourceProvider sourceProvider) { - return new SearchLookup(this, sourceProvider, fieldChain); - } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java index 8b63a146f2e5d..112f101ad842b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java @@ -33,7 +33,6 @@ import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NestedLookup; -import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; @@ -339,16 +338,7 @@ public MappedFieldType.FieldExtractPreference fieldExtractPreference() { @Override public SearchLookup lookup() { - boolean syntheticSource = SourceFieldMapper.isSynthetic(indexSettings()); - var searchLookup = ctx.lookup(); - if (syntheticSource) { - // in the context of scripts and when synthetic source is used the search lookup can't always be reused between - // users of SearchLookup. This is only an issue when scripts fallback to _source, but since we can't always - // accurately determine whether a script uses _source, we should do this for all script usages. - // This lookup() method is only invoked for scripts / runtime fields, so it is ok to do here. - searchLookup = searchLookup.swapSourceProvider(ctx.createSourceProvider()); - } - return searchLookup; + return ctx.lookup(); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ReinitializingSourceProvider.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ReinitializingSourceProvider.java index 8dee3478b3b64..61ac67674b252 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ReinitializingSourceProvider.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ReinitializingSourceProvider.java @@ -28,10 +28,6 @@ final class ReinitializingSourceProvider implements SourceProvider { private PerThreadSourceProvider perThreadProvider; private final Supplier sourceProviderFactory; - // Keeping track of last seen doc and if current doc is before last seen doc then source provider is initialized: - // (when source mode is synthetic then _source is read from doc values and doc values don't support going backwards) - private int lastSeenDocId; - ReinitializingSourceProvider(Supplier sourceProviderFactory) { this.sourceProviderFactory = sourceProviderFactory; } @@ -40,15 +36,25 @@ final class ReinitializingSourceProvider implements SourceProvider { public Source getSource(LeafReaderContext ctx, int doc) throws IOException { var currentThread = Thread.currentThread(); PerThreadSourceProvider provider = perThreadProvider; - if (provider == null || provider.creatingThread != currentThread || doc < lastSeenDocId) { + if (provider == null || provider.creatingThread != currentThread || doc < provider.lastSeenDocId) { provider = new PerThreadSourceProvider(sourceProviderFactory.get(), currentThread); this.perThreadProvider = provider; } - lastSeenDocId = doc; + provider.lastSeenDocId = doc; return provider.source.getSource(ctx, doc); } - private record PerThreadSourceProvider(SourceProvider source, Thread creatingThread) { + private static final class PerThreadSourceProvider { + final SourceProvider source; + final Thread creatingThread; + // Keeping track of last seen doc and if current doc is before last seen doc then source provider is initialized: + // (when source mode is synthetic then _source is read from doc values and doc values don't support going backwards) + int lastSeenDocId; + + private PerThreadSourceProvider(SourceProvider source, Thread creatingThread) { + this.source = source; + this.creatingThread = creatingThread; + } } } From d85b90ad8c76d4b6b06af9c26c0a0c99aa7ab463 Mon Sep 17 00:00:00 2001 From: Pete Gillin Date: Tue, 14 Jan 2025 10:34:46 +0000 Subject: [PATCH 14/31] Remove unfreeze REST endpoint (#119227) This adds a sentence to `redirects.asciidoc` explaining what frozen indices were - otherwise, everything will point to the message about the unfreeze API having gone away, which is not very helpful. Some cross-references are updated to point to this rather than to the notice about the removal of the unfreeze API. ES-9736 #comment Removed `_unfreeze` REST endpoint in https://github.com/elastic/elasticsearch/pull/119227 --- docs/changelog/119227.yaml | 13 ++++ docs/reference/indices.asciidoc | 2 - docs/reference/indices/apis/unfreeze.asciidoc | 61 ----------------- docs/reference/indices/index-mgmt.asciidoc | 2 +- docs/reference/redirects.asciidoc | 16 +++-- docs/reference/sql/language/indices.asciidoc | 2 +- .../rest-api-spec/api/indices.unfreeze.json | 67 ------------------- .../xpack/frozen/FrozenIndices.java | 29 -------- .../rest/action/RestFreezeIndexAction.java | 67 ------------------- 9 files changed, 27 insertions(+), 232 deletions(-) create mode 100644 docs/changelog/119227.yaml delete mode 100644 docs/reference/indices/apis/unfreeze.asciidoc delete mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/indices.unfreeze.json delete mode 100644 x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/rest/action/RestFreezeIndexAction.java diff --git a/docs/changelog/119227.yaml b/docs/changelog/119227.yaml new file mode 100644 index 0000000000000..1e3d4f97a3d27 --- /dev/null +++ b/docs/changelog/119227.yaml @@ -0,0 +1,13 @@ +pr: 119227 +summary: Remove unfreeze REST endpoint +area: Indices APIs +type: breaking +issues: [] +breaking: + title: Remove unfreeze REST endpoint + area: REST API + details: >- + The `/{index}/_unfreeze` REST endpoint is no longer supported. This API was deprecated, and the corresponding + `/{index}/_freeze` endpoint was removed in 8.0. + impact: None, since it is not possible to have a frozen index in a version which is readable by Elasticsearch 9.0 + notable: false diff --git a/docs/reference/indices.asciidoc b/docs/reference/indices.asciidoc index ca7de396147a8..b6b82422cbb4a 100644 --- a/docs/reference/indices.asciidoc +++ b/docs/reference/indices.asciidoc @@ -24,7 +24,6 @@ index settings, aliases, mappings, and index templates. * <> * <> * <> -* <> * <> * <> * <> @@ -143,6 +142,5 @@ include::indices/shrink-index.asciidoc[] include::indices/simulate-index.asciidoc[] include::indices/simulate-template.asciidoc[] include::indices/split-index.asciidoc[] -include::indices/apis/unfreeze.asciidoc[] include::indices/update-settings.asciidoc[] include::indices/put-mapping.asciidoc[] diff --git a/docs/reference/indices/apis/unfreeze.asciidoc b/docs/reference/indices/apis/unfreeze.asciidoc deleted file mode 100644 index 5d04d44db7443..0000000000000 --- a/docs/reference/indices/apis/unfreeze.asciidoc +++ /dev/null @@ -1,61 +0,0 @@ -[role="xpack"] -[[unfreeze-index-api]] -=== Unfreeze index API -++++ -Unfreeze index -++++ - -[WARNING] -.Deprecated in 7.14 -==== -In 8.0, we removed the ability to freeze an index. In previous versions, -freezing an index reduced its memory overhead. However, frozen indices are no -longer useful due to -https://www.elastic.co/blog/significantly-decrease-your-elasticsearch-heap-memory-usage[recent -improvements in heap memory usage]. -You can use this API to unfreeze indices that were frozen in 7.x. Frozen indices -are not related to the frozen data tier. -==== - -.New API reference -[sidebar] --- -For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. --- - -Unfreezes an index. - -[[unfreeze-index-api-request]] -==== {api-request-title} - -`POST //_unfreeze` - -[[unfreeze-index-api-prereqs]] -==== {api-prereq-title} - -* If the {es} {security-features} are enabled, you must have the `manage` -<> for the target index or index alias. - -[[unfreeze-index-api-desc]] -==== {api-description-title} - -When a frozen index is unfrozen, the index goes through the normal recovery -process and becomes writeable again. - -[[unfreeze-index-api-path-parms]] -==== {api-path-parms-title} - -``:: - (Required, string) Identifier for the index. - -[[unfreeze-index-api-examples]] -==== {api-examples-title} - -The following example unfreezes an index: - -[source,console] --------------------------------------------------- -POST /my-index-000001/_unfreeze --------------------------------------------------- -// TEST[s/^/PUT my-index-000001\n/] -// TEST[skip:unable to ignore deprecation warning] diff --git a/docs/reference/indices/index-mgmt.asciidoc b/docs/reference/indices/index-mgmt.asciidoc index 73643dbfd4b3b..131bc79faa40c 100644 --- a/docs/reference/indices/index-mgmt.asciidoc +++ b/docs/reference/indices/index-mgmt.asciidoc @@ -43,7 +43,7 @@ For more information on managing indices, refer to <>. * To filter the list of indices, use the search bar or click a badge. Badges indicate if an index is a <>, a -<>, or <>. +<>, or <>. * To drill down into the index <>, <>, and statistics, diff --git a/docs/reference/redirects.asciidoc b/docs/reference/redirects.asciidoc index c3bf84fa600d2..9c0f0092214ed 100644 --- a/docs/reference/redirects.asciidoc +++ b/docs/reference/redirects.asciidoc @@ -156,10 +156,16 @@ See <>. The freeze index API was removed in 8.0. // tag::frozen-removal-explanation[] Frozen indices are no longer useful due to -https://www.elastic.co/blog/significantly-decrease-your-elasticsearch-heap-memory-usage[recent -improvements in heap memory usage]. +https://www.elastic.co/blog/significantly-decrease-your-elasticsearch-heap-memory-usage[improvements +in heap memory usage]. // end::frozen-removal-explanation[] +[role="exclude",id="unfreeze-index-api"] +=== Unfreeze index API + +The unfreeze index API was removed in 9.0. +include::redirects.asciidoc[tag=frozen-removal-explanation] + [role="exclude",id="ilm-freeze"] === Freeze {ilm-init} action @@ -1749,8 +1755,10 @@ See <>. === Frozen indices // tag::frozen-index-redirect[] - -For API documentation, see <>. +Older versions of {es} provided the option to reduce the amount of data kept in memory for an index, at the expense of +increasing search latency. This was known as 'freezing' the index. +include::redirects.asciidoc[tag=frozen-removal-explanation] +The freeze index API was removed in 8.0, and the unfreeze index API was removed in 9.0. // end::frozen-index-redirect[] [role="exclude",id="best_practices"] diff --git a/docs/reference/sql/language/indices.asciidoc b/docs/reference/sql/language/indices.asciidoc index 1dee7f0840ade..1912a020ab0be 100644 --- a/docs/reference/sql/language/indices.asciidoc +++ b/docs/reference/sql/language/indices.asciidoc @@ -100,7 +100,7 @@ requires the keyword `LIKE` for SQL `LIKE` pattern. [[sql-index-frozen]] === Frozen Indices -By default, {es-sql} doesn't search <>. To +By default, {es-sql} doesn't search <>. To search frozen indices, use one of the following features: dedicated configuration parameter:: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.unfreeze.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.unfreeze.json deleted file mode 100644 index 2327519ff2816..0000000000000 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.unfreeze.json +++ /dev/null @@ -1,67 +0,0 @@ -{ - "indices.unfreeze":{ - "documentation":{ - "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/unfreeze-index-api.html", - "description":"Unfreezes an index. When a frozen index is unfrozen, the index goes through the normal recovery process and becomes writeable again." - }, - "stability":"stable", - "visibility":"public", - "headers":{ - "accept": [ "application/json"] - }, - "url":{ - "paths":[ - { - "path":"/{index}/_unfreeze", - "methods":[ - "POST" - ], - "parts":{ - "index":{ - "type":"string", - "description":"The name of the index to unfreeze" - } - }, - "deprecated":{ - "version":"7.14.0", - "description":"Frozen indices are deprecated because they provide no benefit given improvements in heap memory utilization. They will be removed in a future release." - } - } - ] - }, - "params":{ - "timeout":{ - "type":"time", - "description":"Explicit operation timeout" - }, - "master_timeout":{ - "type":"time", - "description":"Specify timeout for connection to master" - }, - "ignore_unavailable":{ - "type":"boolean", - "description":"Whether specified concrete indices should be ignored when unavailable (missing or closed)" - }, - "allow_no_indices":{ - "type":"boolean", - "description":"Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)" - }, - "expand_wildcards":{ - "type":"enum", - "options":[ - "open", - "closed", - "hidden", - "none", - "all" - ], - "default":"closed", - "description":"Whether to expand wildcard expression to concrete indices that are open, closed or both." - }, - "wait_for_active_shards":{ - "type":"string", - "description":"Sets the number of active shards to wait for before the operation returns." - } - } - } -} diff --git a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/FrozenIndices.java b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/FrozenIndices.java index 05b75fe6b01ca..0b106f6e304da 100644 --- a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/FrozenIndices.java +++ b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/FrozenIndices.java @@ -8,36 +8,22 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.settings.ClusterSettings; -import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.settings.SettingsFilter; -import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.EngineFactory; import org.elasticsearch.index.engine.frozen.FrozenEngine; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.EnginePlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.rest.RestController; -import org.elasticsearch.rest.RestHandler; import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; import org.elasticsearch.xpack.core.frozen.action.FreezeIndexAction; import org.elasticsearch.xpack.frozen.action.TransportFreezeIndexAction; -import org.elasticsearch.xpack.frozen.rest.action.RestFreezeIndexAction; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Optional; -import java.util.function.Predicate; -import java.util.function.Supplier; public class FrozenIndices extends Plugin implements ActionPlugin, EnginePlugin { @@ -63,19 +49,4 @@ public List> getSettings() { actions.add(new ActionHandler<>(FreezeIndexAction.INSTANCE, TransportFreezeIndexAction.class)); return actions; } - - @Override - public List getRestHandlers( - Settings settings, - NamedWriteableRegistry namedWriteableRegistry, - RestController restController, - ClusterSettings clusterSettings, - IndexScopedSettings indexScopedSettings, - SettingsFilter settingsFilter, - IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster, - Predicate clusterSupportsFeature - ) { - return Collections.singletonList(new RestFreezeIndexAction()); - } } diff --git a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/rest/action/RestFreezeIndexAction.java b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/rest/action/RestFreezeIndexAction.java deleted file mode 100644 index f07d17fee87f5..0000000000000 --- a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/rest/action/RestFreezeIndexAction.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.frozen.rest.action; - -import org.elasticsearch.action.support.ActiveShardCount; -import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.client.internal.node.NodeClient; -import org.elasticsearch.common.Strings; -import org.elasticsearch.core.UpdateForV9; -import org.elasticsearch.protocol.xpack.frozen.FreezeRequest; -import org.elasticsearch.rest.BaseRestHandler; -import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.action.RestToXContentListener; -import org.elasticsearch.xpack.core.frozen.action.FreezeIndexAction; - -import java.util.List; - -import static org.elasticsearch.rest.RestRequest.Method.POST; -import static org.elasticsearch.rest.RestUtils.getAckTimeout; -import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout; - -public final class RestFreezeIndexAction extends BaseRestHandler { - - private static final String FREEZE_REMOVED = "It is no longer possible to freeze indices, but existing frozen indices can still be " - + "unfrozen"; - - private static final String UNFREEZE_DEPRECATED = "Frozen indices are deprecated because they provide no benefit given improvements " - + "in heap memory utilization. They will be removed in a future release."; - - @UpdateForV9(owner = UpdateForV9.Owner.DISTRIBUTED_INDEXING) - // these routes were ".deprecated" in RestApiVersion.V_8 which will require use of REST API compatibility headers to access - // this API in v9. It is unclear if this was intentional for v9, and the code has been updated to ".deprecateAndKeep" which will - // continue to emit deprecations warnings but will not require any special headers to access the API in v9. - // Please review and update the code and tests as needed. The original code remains commented out below for reference. - @Override - public List routes() { - return List.of( - // Route.builder(POST, "/{index}/_unfreeze").deprecated(UNFREEZE_DEPRECATED, RestApiVersion.V_8).build() - Route.builder(POST, "/{index}/_unfreeze").deprecateAndKeep(UNFREEZE_DEPRECATED).build() - ); - } - - @Override - protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { - final var freezeRequest = new FreezeRequest( - getMasterNodeTimeout(request), - getAckTimeout(request), - Strings.splitStringByCommaToArray(request.param("index")) - ); - freezeRequest.indicesOptions(IndicesOptions.fromRequest(request, freezeRequest.indicesOptions())); - String waitForActiveShards = request.param("wait_for_active_shards"); - if (waitForActiveShards != null) { - freezeRequest.waitForActiveShards(ActiveShardCount.parseString(waitForActiveShards)); - } - freezeRequest.setFreeze(false); - return channel -> client.execute(FreezeIndexAction.INSTANCE, freezeRequest, new RestToXContentListener<>(channel)); - } - - @Override - public String getName() { - return "freeze_index"; - } -} From 60a54e6ab702ef2d6cf10ac0c0d5d4a4c1b528d0 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 14 Jan 2025 21:38:09 +1100 Subject: [PATCH 15/31] [Test] Upgrade minio docker image used for tests (#120086) This PR upgrades the minio docker image from RELEASE.2021-03-01T04-20-55Z which is 3+ years old to the latest RELEASE.2024-12-18T13-15-44Z. Relates: #118548 --- .../s3/RepositoryS3MinioBasicCredentialsRestIT.java | 6 ++++-- .../test/fixtures/minio/MinioTestContainer.java | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3MinioBasicCredentialsRestIT.java b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3MinioBasicCredentialsRestIT.java index 93915e8491d5b..3d7c8dd150610 100644 --- a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3MinioBasicCredentialsRestIT.java +++ b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3MinioBasicCredentialsRestIT.java @@ -19,13 +19,15 @@ import org.junit.rules.RuleChain; import org.junit.rules.TestRule; +import java.util.Locale; + @ThreadLeakFilters(filters = { TestContainersThreadFilter.class }) @ThreadLeakScope(ThreadLeakScope.Scope.NONE) // https://github.com/elastic/elasticsearch/issues/102482 public class RepositoryS3MinioBasicCredentialsRestIT extends AbstractRepositoryS3RestTestCase { - private static final String PREFIX = getIdentifierPrefix("RepositoryS3MinioBasicCredentialsRestIT"); + private static final String PREFIX = getIdentifierPrefix("RepositoryS3MinioBasicCredentialsRestIT").toLowerCase(Locale.ROOT); private static final String BUCKET = PREFIX + "bucket"; - private static final String BASE_PATH = PREFIX + "base_path"; + private static final String BASE_PATH = PREFIX + "base-path"; private static final String ACCESS_KEY = PREFIX + "access-key"; private static final String SECRET_KEY = PREFIX + "secret-key"; private static final String CLIENT = "minio_client"; diff --git a/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/minio/MinioTestContainer.java b/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/minio/MinioTestContainer.java index 3ee18d71a5a79..56e702d2f76a0 100644 --- a/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/minio/MinioTestContainer.java +++ b/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/minio/MinioTestContainer.java @@ -15,7 +15,7 @@ public final class MinioTestContainer extends DockerEnvironmentAwareTestContainer { private static final int servicePort = 9000; - public static final String DOCKER_BASE_IMAGE = "minio/minio:RELEASE.2021-03-01T04-20-55Z"; + public static final String DOCKER_BASE_IMAGE = "minio/minio:RELEASE.2024-12-18T13-15-44Z"; private final boolean enabled; public MinioTestContainer(boolean enabled, String accessKey, String secretKey, String bucketName) { From 212e16bc3a5f6bef412f2ba158d5ace7f62ea300 Mon Sep 17 00:00:00 2001 From: Bogdan Pintea Date: Tue, 14 Jan 2025 12:06:35 +0100 Subject: [PATCH 16/31] ESQL: Drop support for brackets from METADATA syntax (#119846) This drops the deprecated support for having the METADATA clause provided within square brakets. Fixes #115401 --- docs/changelog/119846.yaml | 12 + .../esql/src/main/antlr/EsqlBaseParser.g4 | 9 - .../xpack/esql/parser/EsqlBaseParser.interp | 4 +- .../xpack/esql/parser/EsqlBaseParser.java | 1799 ++++++++--------- .../parser/EsqlBaseParserBaseListener.java | 24 - .../parser/EsqlBaseParserBaseVisitor.java | 14 - .../esql/parser/EsqlBaseParserListener.java | 20 - .../esql/parser/EsqlBaseParserVisitor.java | 12 - .../xpack/esql/parser/LogicalPlanBuilder.java | 18 +- .../esql/parser/StatementParserTests.java | 19 +- .../test/querying_cluster/80_esql.yml | 6 +- 11 files changed, 863 insertions(+), 1074 deletions(-) create mode 100644 docs/changelog/119846.yaml diff --git a/docs/changelog/119846.yaml b/docs/changelog/119846.yaml new file mode 100644 index 0000000000000..9e7d99fe1be13 --- /dev/null +++ b/docs/changelog/119846.yaml @@ -0,0 +1,12 @@ +pr: 119846 +summary: Drop support for brackets from METADATA syntax +area: ES|QL +type: deprecation +issues: + - 115401 +deprecation: + title: Drop support for brackets from METADATA syntax + area: ES|QL + details: Please describe the details of this change for the release notes. You can + use asciidoc. + impact: Please describe the impact of this change to users diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 index efc2e36609902..4d4cb7d2caac4 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 @@ -143,18 +143,9 @@ indexString ; metadata - : metadataOption - | deprecated_metadata - ; - -metadataOption : METADATA UNQUOTED_SOURCE (COMMA UNQUOTED_SOURCE)* ; -deprecated_metadata - : OPENING_BRACKET metadataOption CLOSING_BRACKET - ; - metricsCommand : DEV_METRICS indexPattern (COMMA indexPattern)* aggregates=aggFields? (BY grouping=fields)? ; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp index c5b37fa411f65..492df7fbc1608 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp @@ -283,8 +283,6 @@ indexPattern clusterString indexString metadata -metadataOption -deprecated_metadata metricsCommand evalCommand statsCommand @@ -330,4 +328,4 @@ joinPredicate atn: -[4, 1, 128, 639, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 2, 64, 7, 64, 2, 65, 7, 65, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 142, 8, 1, 10, 1, 12, 1, 145, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 153, 8, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 173, 8, 3, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 185, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 192, 8, 5, 10, 5, 12, 5, 195, 9, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 202, 8, 5, 1, 5, 1, 5, 1, 5, 3, 5, 207, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 215, 8, 5, 10, 5, 12, 5, 218, 9, 5, 1, 6, 1, 6, 3, 6, 222, 8, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 3, 6, 229, 8, 6, 1, 6, 1, 6, 1, 6, 3, 6, 234, 8, 6, 1, 7, 1, 7, 1, 7, 3, 7, 239, 8, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 3, 8, 249, 8, 8, 1, 9, 1, 9, 1, 9, 1, 9, 3, 9, 255, 8, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 5, 9, 263, 8, 9, 10, 9, 12, 9, 266, 9, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 3, 10, 276, 8, 10, 1, 10, 1, 10, 1, 10, 5, 10, 281, 8, 10, 10, 10, 12, 10, 284, 9, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 5, 11, 292, 8, 11, 10, 11, 12, 11, 295, 9, 11, 3, 11, 297, 8, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 5, 15, 311, 8, 15, 10, 15, 12, 15, 314, 9, 15, 1, 16, 1, 16, 1, 16, 3, 16, 319, 8, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 5, 17, 327, 8, 17, 10, 17, 12, 17, 330, 9, 17, 1, 17, 3, 17, 333, 8, 17, 1, 18, 1, 18, 1, 18, 3, 18, 338, 8, 18, 1, 18, 1, 18, 1, 19, 1, 19, 1, 20, 1, 20, 1, 21, 1, 21, 3, 21, 348, 8, 21, 1, 22, 1, 22, 1, 22, 1, 22, 5, 22, 354, 8, 22, 10, 22, 12, 22, 357, 9, 22, 1, 23, 1, 23, 1, 23, 1, 23, 1, 24, 1, 24, 1, 24, 1, 24, 5, 24, 367, 8, 24, 10, 24, 12, 24, 370, 9, 24, 1, 24, 3, 24, 373, 8, 24, 1, 24, 1, 24, 3, 24, 377, 8, 24, 1, 25, 1, 25, 1, 25, 1, 26, 1, 26, 3, 26, 384, 8, 26, 1, 26, 1, 26, 3, 26, 388, 8, 26, 1, 27, 1, 27, 1, 27, 5, 27, 393, 8, 27, 10, 27, 12, 27, 396, 9, 27, 1, 28, 1, 28, 1, 28, 3, 28, 401, 8, 28, 1, 29, 1, 29, 1, 29, 5, 29, 406, 8, 29, 10, 29, 12, 29, 409, 9, 29, 1, 30, 1, 30, 1, 30, 5, 30, 414, 8, 30, 10, 30, 12, 30, 417, 9, 30, 1, 31, 1, 31, 1, 31, 5, 31, 422, 8, 31, 10, 31, 12, 31, 425, 9, 31, 1, 32, 1, 32, 1, 33, 1, 33, 1, 33, 3, 33, 432, 8, 33, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 447, 8, 34, 10, 34, 12, 34, 450, 9, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 458, 8, 34, 10, 34, 12, 34, 461, 9, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 469, 8, 34, 10, 34, 12, 34, 472, 9, 34, 1, 34, 1, 34, 3, 34, 476, 8, 34, 1, 35, 1, 35, 3, 35, 480, 8, 35, 1, 36, 1, 36, 1, 36, 3, 36, 485, 8, 36, 1, 37, 1, 37, 1, 37, 1, 38, 1, 38, 1, 38, 1, 38, 5, 38, 494, 8, 38, 10, 38, 12, 38, 497, 9, 38, 1, 39, 1, 39, 3, 39, 501, 8, 39, 1, 39, 1, 39, 3, 39, 505, 8, 39, 1, 40, 1, 40, 1, 40, 1, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 42, 1, 42, 5, 42, 517, 8, 42, 10, 42, 12, 42, 520, 9, 42, 1, 43, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 44, 1, 44, 3, 44, 530, 8, 44, 1, 45, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 47, 5, 47, 542, 8, 47, 10, 47, 12, 47, 545, 9, 47, 1, 48, 1, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 50, 1, 50, 3, 50, 555, 8, 50, 1, 51, 3, 51, 558, 8, 51, 1, 51, 1, 51, 1, 52, 3, 52, 563, 8, 52, 1, 52, 1, 52, 1, 53, 1, 53, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 58, 3, 58, 585, 8, 58, 1, 58, 1, 58, 1, 58, 1, 58, 5, 58, 591, 8, 58, 10, 58, 12, 58, 594, 9, 58, 3, 58, 596, 8, 58, 1, 59, 1, 59, 1, 59, 3, 59, 601, 8, 59, 1, 59, 1, 59, 1, 60, 1, 60, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 1, 61, 3, 61, 614, 8, 61, 1, 62, 3, 62, 617, 8, 62, 1, 62, 1, 62, 1, 62, 1, 62, 1, 63, 1, 63, 1, 63, 3, 63, 626, 8, 63, 1, 64, 1, 64, 1, 64, 1, 64, 5, 64, 632, 8, 64, 10, 64, 12, 64, 635, 9, 64, 1, 65, 1, 65, 1, 65, 0, 4, 2, 10, 18, 20, 66, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126, 128, 130, 0, 9, 1, 0, 64, 65, 1, 0, 66, 68, 2, 0, 30, 30, 81, 81, 1, 0, 72, 73, 2, 0, 35, 35, 40, 40, 2, 0, 43, 43, 46, 46, 2, 0, 42, 42, 56, 56, 2, 0, 57, 57, 59, 63, 1, 0, 22, 24, 665, 0, 132, 1, 0, 0, 0, 2, 135, 1, 0, 0, 0, 4, 152, 1, 0, 0, 0, 6, 172, 1, 0, 0, 0, 8, 174, 1, 0, 0, 0, 10, 206, 1, 0, 0, 0, 12, 233, 1, 0, 0, 0, 14, 235, 1, 0, 0, 0, 16, 248, 1, 0, 0, 0, 18, 254, 1, 0, 0, 0, 20, 275, 1, 0, 0, 0, 22, 285, 1, 0, 0, 0, 24, 300, 1, 0, 0, 0, 26, 302, 1, 0, 0, 0, 28, 304, 1, 0, 0, 0, 30, 307, 1, 0, 0, 0, 32, 318, 1, 0, 0, 0, 34, 322, 1, 0, 0, 0, 36, 337, 1, 0, 0, 0, 38, 341, 1, 0, 0, 0, 40, 343, 1, 0, 0, 0, 42, 347, 1, 0, 0, 0, 44, 349, 1, 0, 0, 0, 46, 358, 1, 0, 0, 0, 48, 362, 1, 0, 0, 0, 50, 378, 1, 0, 0, 0, 52, 381, 1, 0, 0, 0, 54, 389, 1, 0, 0, 0, 56, 397, 1, 0, 0, 0, 58, 402, 1, 0, 0, 0, 60, 410, 1, 0, 0, 0, 62, 418, 1, 0, 0, 0, 64, 426, 1, 0, 0, 0, 66, 431, 1, 0, 0, 0, 68, 475, 1, 0, 0, 0, 70, 479, 1, 0, 0, 0, 72, 484, 1, 0, 0, 0, 74, 486, 1, 0, 0, 0, 76, 489, 1, 0, 0, 0, 78, 498, 1, 0, 0, 0, 80, 506, 1, 0, 0, 0, 82, 509, 1, 0, 0, 0, 84, 512, 1, 0, 0, 0, 86, 521, 1, 0, 0, 0, 88, 525, 1, 0, 0, 0, 90, 531, 1, 0, 0, 0, 92, 535, 1, 0, 0, 0, 94, 538, 1, 0, 0, 0, 96, 546, 1, 0, 0, 0, 98, 550, 1, 0, 0, 0, 100, 554, 1, 0, 0, 0, 102, 557, 1, 0, 0, 0, 104, 562, 1, 0, 0, 0, 106, 566, 1, 0, 0, 0, 108, 568, 1, 0, 0, 0, 110, 570, 1, 0, 0, 0, 112, 573, 1, 0, 0, 0, 114, 577, 1, 0, 0, 0, 116, 580, 1, 0, 0, 0, 118, 600, 1, 0, 0, 0, 120, 604, 1, 0, 0, 0, 122, 609, 1, 0, 0, 0, 124, 616, 1, 0, 0, 0, 126, 622, 1, 0, 0, 0, 128, 627, 1, 0, 0, 0, 130, 636, 1, 0, 0, 0, 132, 133, 3, 2, 1, 0, 133, 134, 5, 0, 0, 1, 134, 1, 1, 0, 0, 0, 135, 136, 6, 1, -1, 0, 136, 137, 3, 4, 2, 0, 137, 143, 1, 0, 0, 0, 138, 139, 10, 1, 0, 0, 139, 140, 5, 29, 0, 0, 140, 142, 3, 6, 3, 0, 141, 138, 1, 0, 0, 0, 142, 145, 1, 0, 0, 0, 143, 141, 1, 0, 0, 0, 143, 144, 1, 0, 0, 0, 144, 3, 1, 0, 0, 0, 145, 143, 1, 0, 0, 0, 146, 153, 3, 110, 55, 0, 147, 153, 3, 34, 17, 0, 148, 153, 3, 28, 14, 0, 149, 153, 3, 114, 57, 0, 150, 151, 4, 2, 1, 0, 151, 153, 3, 48, 24, 0, 152, 146, 1, 0, 0, 0, 152, 147, 1, 0, 0, 0, 152, 148, 1, 0, 0, 0, 152, 149, 1, 0, 0, 0, 152, 150, 1, 0, 0, 0, 153, 5, 1, 0, 0, 0, 154, 173, 3, 50, 25, 0, 155, 173, 3, 8, 4, 0, 156, 173, 3, 80, 40, 0, 157, 173, 3, 74, 37, 0, 158, 173, 3, 52, 26, 0, 159, 173, 3, 76, 38, 0, 160, 173, 3, 82, 41, 0, 161, 173, 3, 84, 42, 0, 162, 173, 3, 88, 44, 0, 163, 173, 3, 90, 45, 0, 164, 173, 3, 116, 58, 0, 165, 173, 3, 92, 46, 0, 166, 167, 4, 3, 2, 0, 167, 173, 3, 122, 61, 0, 168, 169, 4, 3, 3, 0, 169, 173, 3, 120, 60, 0, 170, 171, 4, 3, 4, 0, 171, 173, 3, 124, 62, 0, 172, 154, 1, 0, 0, 0, 172, 155, 1, 0, 0, 0, 172, 156, 1, 0, 0, 0, 172, 157, 1, 0, 0, 0, 172, 158, 1, 0, 0, 0, 172, 159, 1, 0, 0, 0, 172, 160, 1, 0, 0, 0, 172, 161, 1, 0, 0, 0, 172, 162, 1, 0, 0, 0, 172, 163, 1, 0, 0, 0, 172, 164, 1, 0, 0, 0, 172, 165, 1, 0, 0, 0, 172, 166, 1, 0, 0, 0, 172, 168, 1, 0, 0, 0, 172, 170, 1, 0, 0, 0, 173, 7, 1, 0, 0, 0, 174, 175, 5, 16, 0, 0, 175, 176, 3, 10, 5, 0, 176, 9, 1, 0, 0, 0, 177, 178, 6, 5, -1, 0, 178, 179, 5, 49, 0, 0, 179, 207, 3, 10, 5, 8, 180, 207, 3, 16, 8, 0, 181, 207, 3, 12, 6, 0, 182, 184, 3, 16, 8, 0, 183, 185, 5, 49, 0, 0, 184, 183, 1, 0, 0, 0, 184, 185, 1, 0, 0, 0, 185, 186, 1, 0, 0, 0, 186, 187, 5, 44, 0, 0, 187, 188, 5, 48, 0, 0, 188, 193, 3, 16, 8, 0, 189, 190, 5, 39, 0, 0, 190, 192, 3, 16, 8, 0, 191, 189, 1, 0, 0, 0, 192, 195, 1, 0, 0, 0, 193, 191, 1, 0, 0, 0, 193, 194, 1, 0, 0, 0, 194, 196, 1, 0, 0, 0, 195, 193, 1, 0, 0, 0, 196, 197, 5, 55, 0, 0, 197, 207, 1, 0, 0, 0, 198, 199, 3, 16, 8, 0, 199, 201, 5, 45, 0, 0, 200, 202, 5, 49, 0, 0, 201, 200, 1, 0, 0, 0, 201, 202, 1, 0, 0, 0, 202, 203, 1, 0, 0, 0, 203, 204, 5, 50, 0, 0, 204, 207, 1, 0, 0, 0, 205, 207, 3, 14, 7, 0, 206, 177, 1, 0, 0, 0, 206, 180, 1, 0, 0, 0, 206, 181, 1, 0, 0, 0, 206, 182, 1, 0, 0, 0, 206, 198, 1, 0, 0, 0, 206, 205, 1, 0, 0, 0, 207, 216, 1, 0, 0, 0, 208, 209, 10, 5, 0, 0, 209, 210, 5, 34, 0, 0, 210, 215, 3, 10, 5, 6, 211, 212, 10, 4, 0, 0, 212, 213, 5, 52, 0, 0, 213, 215, 3, 10, 5, 5, 214, 208, 1, 0, 0, 0, 214, 211, 1, 0, 0, 0, 215, 218, 1, 0, 0, 0, 216, 214, 1, 0, 0, 0, 216, 217, 1, 0, 0, 0, 217, 11, 1, 0, 0, 0, 218, 216, 1, 0, 0, 0, 219, 221, 3, 16, 8, 0, 220, 222, 5, 49, 0, 0, 221, 220, 1, 0, 0, 0, 221, 222, 1, 0, 0, 0, 222, 223, 1, 0, 0, 0, 223, 224, 5, 47, 0, 0, 224, 225, 3, 106, 53, 0, 225, 234, 1, 0, 0, 0, 226, 228, 3, 16, 8, 0, 227, 229, 5, 49, 0, 0, 228, 227, 1, 0, 0, 0, 228, 229, 1, 0, 0, 0, 229, 230, 1, 0, 0, 0, 230, 231, 5, 54, 0, 0, 231, 232, 3, 106, 53, 0, 232, 234, 1, 0, 0, 0, 233, 219, 1, 0, 0, 0, 233, 226, 1, 0, 0, 0, 234, 13, 1, 0, 0, 0, 235, 238, 3, 58, 29, 0, 236, 237, 5, 37, 0, 0, 237, 239, 3, 26, 13, 0, 238, 236, 1, 0, 0, 0, 238, 239, 1, 0, 0, 0, 239, 240, 1, 0, 0, 0, 240, 241, 5, 38, 0, 0, 241, 242, 3, 68, 34, 0, 242, 15, 1, 0, 0, 0, 243, 249, 3, 18, 9, 0, 244, 245, 3, 18, 9, 0, 245, 246, 3, 108, 54, 0, 246, 247, 3, 18, 9, 0, 247, 249, 1, 0, 0, 0, 248, 243, 1, 0, 0, 0, 248, 244, 1, 0, 0, 0, 249, 17, 1, 0, 0, 0, 250, 251, 6, 9, -1, 0, 251, 255, 3, 20, 10, 0, 252, 253, 7, 0, 0, 0, 253, 255, 3, 18, 9, 3, 254, 250, 1, 0, 0, 0, 254, 252, 1, 0, 0, 0, 255, 264, 1, 0, 0, 0, 256, 257, 10, 2, 0, 0, 257, 258, 7, 1, 0, 0, 258, 263, 3, 18, 9, 3, 259, 260, 10, 1, 0, 0, 260, 261, 7, 0, 0, 0, 261, 263, 3, 18, 9, 2, 262, 256, 1, 0, 0, 0, 262, 259, 1, 0, 0, 0, 263, 266, 1, 0, 0, 0, 264, 262, 1, 0, 0, 0, 264, 265, 1, 0, 0, 0, 265, 19, 1, 0, 0, 0, 266, 264, 1, 0, 0, 0, 267, 268, 6, 10, -1, 0, 268, 276, 3, 68, 34, 0, 269, 276, 3, 58, 29, 0, 270, 276, 3, 22, 11, 0, 271, 272, 5, 48, 0, 0, 272, 273, 3, 10, 5, 0, 273, 274, 5, 55, 0, 0, 274, 276, 1, 0, 0, 0, 275, 267, 1, 0, 0, 0, 275, 269, 1, 0, 0, 0, 275, 270, 1, 0, 0, 0, 275, 271, 1, 0, 0, 0, 276, 282, 1, 0, 0, 0, 277, 278, 10, 1, 0, 0, 278, 279, 5, 37, 0, 0, 279, 281, 3, 26, 13, 0, 280, 277, 1, 0, 0, 0, 281, 284, 1, 0, 0, 0, 282, 280, 1, 0, 0, 0, 282, 283, 1, 0, 0, 0, 283, 21, 1, 0, 0, 0, 284, 282, 1, 0, 0, 0, 285, 286, 3, 24, 12, 0, 286, 296, 5, 48, 0, 0, 287, 297, 5, 66, 0, 0, 288, 293, 3, 10, 5, 0, 289, 290, 5, 39, 0, 0, 290, 292, 3, 10, 5, 0, 291, 289, 1, 0, 0, 0, 292, 295, 1, 0, 0, 0, 293, 291, 1, 0, 0, 0, 293, 294, 1, 0, 0, 0, 294, 297, 1, 0, 0, 0, 295, 293, 1, 0, 0, 0, 296, 287, 1, 0, 0, 0, 296, 288, 1, 0, 0, 0, 296, 297, 1, 0, 0, 0, 297, 298, 1, 0, 0, 0, 298, 299, 5, 55, 0, 0, 299, 23, 1, 0, 0, 0, 300, 301, 3, 72, 36, 0, 301, 25, 1, 0, 0, 0, 302, 303, 3, 64, 32, 0, 303, 27, 1, 0, 0, 0, 304, 305, 5, 12, 0, 0, 305, 306, 3, 30, 15, 0, 306, 29, 1, 0, 0, 0, 307, 312, 3, 32, 16, 0, 308, 309, 5, 39, 0, 0, 309, 311, 3, 32, 16, 0, 310, 308, 1, 0, 0, 0, 311, 314, 1, 0, 0, 0, 312, 310, 1, 0, 0, 0, 312, 313, 1, 0, 0, 0, 313, 31, 1, 0, 0, 0, 314, 312, 1, 0, 0, 0, 315, 316, 3, 58, 29, 0, 316, 317, 5, 36, 0, 0, 317, 319, 1, 0, 0, 0, 318, 315, 1, 0, 0, 0, 318, 319, 1, 0, 0, 0, 319, 320, 1, 0, 0, 0, 320, 321, 3, 10, 5, 0, 321, 33, 1, 0, 0, 0, 322, 323, 5, 6, 0, 0, 323, 328, 3, 36, 18, 0, 324, 325, 5, 39, 0, 0, 325, 327, 3, 36, 18, 0, 326, 324, 1, 0, 0, 0, 327, 330, 1, 0, 0, 0, 328, 326, 1, 0, 0, 0, 328, 329, 1, 0, 0, 0, 329, 332, 1, 0, 0, 0, 330, 328, 1, 0, 0, 0, 331, 333, 3, 42, 21, 0, 332, 331, 1, 0, 0, 0, 332, 333, 1, 0, 0, 0, 333, 35, 1, 0, 0, 0, 334, 335, 3, 38, 19, 0, 335, 336, 5, 38, 0, 0, 336, 338, 1, 0, 0, 0, 337, 334, 1, 0, 0, 0, 337, 338, 1, 0, 0, 0, 338, 339, 1, 0, 0, 0, 339, 340, 3, 40, 20, 0, 340, 37, 1, 0, 0, 0, 341, 342, 5, 81, 0, 0, 342, 39, 1, 0, 0, 0, 343, 344, 7, 2, 0, 0, 344, 41, 1, 0, 0, 0, 345, 348, 3, 44, 22, 0, 346, 348, 3, 46, 23, 0, 347, 345, 1, 0, 0, 0, 347, 346, 1, 0, 0, 0, 348, 43, 1, 0, 0, 0, 349, 350, 5, 80, 0, 0, 350, 355, 5, 81, 0, 0, 351, 352, 5, 39, 0, 0, 352, 354, 5, 81, 0, 0, 353, 351, 1, 0, 0, 0, 354, 357, 1, 0, 0, 0, 355, 353, 1, 0, 0, 0, 355, 356, 1, 0, 0, 0, 356, 45, 1, 0, 0, 0, 357, 355, 1, 0, 0, 0, 358, 359, 5, 70, 0, 0, 359, 360, 3, 44, 22, 0, 360, 361, 5, 71, 0, 0, 361, 47, 1, 0, 0, 0, 362, 363, 5, 19, 0, 0, 363, 368, 3, 36, 18, 0, 364, 365, 5, 39, 0, 0, 365, 367, 3, 36, 18, 0, 366, 364, 1, 0, 0, 0, 367, 370, 1, 0, 0, 0, 368, 366, 1, 0, 0, 0, 368, 369, 1, 0, 0, 0, 369, 372, 1, 0, 0, 0, 370, 368, 1, 0, 0, 0, 371, 373, 3, 54, 27, 0, 372, 371, 1, 0, 0, 0, 372, 373, 1, 0, 0, 0, 373, 376, 1, 0, 0, 0, 374, 375, 5, 33, 0, 0, 375, 377, 3, 30, 15, 0, 376, 374, 1, 0, 0, 0, 376, 377, 1, 0, 0, 0, 377, 49, 1, 0, 0, 0, 378, 379, 5, 4, 0, 0, 379, 380, 3, 30, 15, 0, 380, 51, 1, 0, 0, 0, 381, 383, 5, 15, 0, 0, 382, 384, 3, 54, 27, 0, 383, 382, 1, 0, 0, 0, 383, 384, 1, 0, 0, 0, 384, 387, 1, 0, 0, 0, 385, 386, 5, 33, 0, 0, 386, 388, 3, 30, 15, 0, 387, 385, 1, 0, 0, 0, 387, 388, 1, 0, 0, 0, 388, 53, 1, 0, 0, 0, 389, 394, 3, 56, 28, 0, 390, 391, 5, 39, 0, 0, 391, 393, 3, 56, 28, 0, 392, 390, 1, 0, 0, 0, 393, 396, 1, 0, 0, 0, 394, 392, 1, 0, 0, 0, 394, 395, 1, 0, 0, 0, 395, 55, 1, 0, 0, 0, 396, 394, 1, 0, 0, 0, 397, 400, 3, 32, 16, 0, 398, 399, 5, 16, 0, 0, 399, 401, 3, 10, 5, 0, 400, 398, 1, 0, 0, 0, 400, 401, 1, 0, 0, 0, 401, 57, 1, 0, 0, 0, 402, 407, 3, 72, 36, 0, 403, 404, 5, 41, 0, 0, 404, 406, 3, 72, 36, 0, 405, 403, 1, 0, 0, 0, 406, 409, 1, 0, 0, 0, 407, 405, 1, 0, 0, 0, 407, 408, 1, 0, 0, 0, 408, 59, 1, 0, 0, 0, 409, 407, 1, 0, 0, 0, 410, 415, 3, 66, 33, 0, 411, 412, 5, 41, 0, 0, 412, 414, 3, 66, 33, 0, 413, 411, 1, 0, 0, 0, 414, 417, 1, 0, 0, 0, 415, 413, 1, 0, 0, 0, 415, 416, 1, 0, 0, 0, 416, 61, 1, 0, 0, 0, 417, 415, 1, 0, 0, 0, 418, 423, 3, 60, 30, 0, 419, 420, 5, 39, 0, 0, 420, 422, 3, 60, 30, 0, 421, 419, 1, 0, 0, 0, 422, 425, 1, 0, 0, 0, 423, 421, 1, 0, 0, 0, 423, 424, 1, 0, 0, 0, 424, 63, 1, 0, 0, 0, 425, 423, 1, 0, 0, 0, 426, 427, 7, 3, 0, 0, 427, 65, 1, 0, 0, 0, 428, 432, 5, 85, 0, 0, 429, 430, 4, 33, 10, 0, 430, 432, 3, 70, 35, 0, 431, 428, 1, 0, 0, 0, 431, 429, 1, 0, 0, 0, 432, 67, 1, 0, 0, 0, 433, 476, 5, 50, 0, 0, 434, 435, 3, 104, 52, 0, 435, 436, 5, 72, 0, 0, 436, 476, 1, 0, 0, 0, 437, 476, 3, 102, 51, 0, 438, 476, 3, 104, 52, 0, 439, 476, 3, 98, 49, 0, 440, 476, 3, 70, 35, 0, 441, 476, 3, 106, 53, 0, 442, 443, 5, 70, 0, 0, 443, 448, 3, 100, 50, 0, 444, 445, 5, 39, 0, 0, 445, 447, 3, 100, 50, 0, 446, 444, 1, 0, 0, 0, 447, 450, 1, 0, 0, 0, 448, 446, 1, 0, 0, 0, 448, 449, 1, 0, 0, 0, 449, 451, 1, 0, 0, 0, 450, 448, 1, 0, 0, 0, 451, 452, 5, 71, 0, 0, 452, 476, 1, 0, 0, 0, 453, 454, 5, 70, 0, 0, 454, 459, 3, 98, 49, 0, 455, 456, 5, 39, 0, 0, 456, 458, 3, 98, 49, 0, 457, 455, 1, 0, 0, 0, 458, 461, 1, 0, 0, 0, 459, 457, 1, 0, 0, 0, 459, 460, 1, 0, 0, 0, 460, 462, 1, 0, 0, 0, 461, 459, 1, 0, 0, 0, 462, 463, 5, 71, 0, 0, 463, 476, 1, 0, 0, 0, 464, 465, 5, 70, 0, 0, 465, 470, 3, 106, 53, 0, 466, 467, 5, 39, 0, 0, 467, 469, 3, 106, 53, 0, 468, 466, 1, 0, 0, 0, 469, 472, 1, 0, 0, 0, 470, 468, 1, 0, 0, 0, 470, 471, 1, 0, 0, 0, 471, 473, 1, 0, 0, 0, 472, 470, 1, 0, 0, 0, 473, 474, 5, 71, 0, 0, 474, 476, 1, 0, 0, 0, 475, 433, 1, 0, 0, 0, 475, 434, 1, 0, 0, 0, 475, 437, 1, 0, 0, 0, 475, 438, 1, 0, 0, 0, 475, 439, 1, 0, 0, 0, 475, 440, 1, 0, 0, 0, 475, 441, 1, 0, 0, 0, 475, 442, 1, 0, 0, 0, 475, 453, 1, 0, 0, 0, 475, 464, 1, 0, 0, 0, 476, 69, 1, 0, 0, 0, 477, 480, 5, 53, 0, 0, 478, 480, 5, 69, 0, 0, 479, 477, 1, 0, 0, 0, 479, 478, 1, 0, 0, 0, 480, 71, 1, 0, 0, 0, 481, 485, 3, 64, 32, 0, 482, 483, 4, 36, 11, 0, 483, 485, 3, 70, 35, 0, 484, 481, 1, 0, 0, 0, 484, 482, 1, 0, 0, 0, 485, 73, 1, 0, 0, 0, 486, 487, 5, 9, 0, 0, 487, 488, 5, 31, 0, 0, 488, 75, 1, 0, 0, 0, 489, 490, 5, 14, 0, 0, 490, 495, 3, 78, 39, 0, 491, 492, 5, 39, 0, 0, 492, 494, 3, 78, 39, 0, 493, 491, 1, 0, 0, 0, 494, 497, 1, 0, 0, 0, 495, 493, 1, 0, 0, 0, 495, 496, 1, 0, 0, 0, 496, 77, 1, 0, 0, 0, 497, 495, 1, 0, 0, 0, 498, 500, 3, 10, 5, 0, 499, 501, 7, 4, 0, 0, 500, 499, 1, 0, 0, 0, 500, 501, 1, 0, 0, 0, 501, 504, 1, 0, 0, 0, 502, 503, 5, 51, 0, 0, 503, 505, 7, 5, 0, 0, 504, 502, 1, 0, 0, 0, 504, 505, 1, 0, 0, 0, 505, 79, 1, 0, 0, 0, 506, 507, 5, 8, 0, 0, 507, 508, 3, 62, 31, 0, 508, 81, 1, 0, 0, 0, 509, 510, 5, 2, 0, 0, 510, 511, 3, 62, 31, 0, 511, 83, 1, 0, 0, 0, 512, 513, 5, 11, 0, 0, 513, 518, 3, 86, 43, 0, 514, 515, 5, 39, 0, 0, 515, 517, 3, 86, 43, 0, 516, 514, 1, 0, 0, 0, 517, 520, 1, 0, 0, 0, 518, 516, 1, 0, 0, 0, 518, 519, 1, 0, 0, 0, 519, 85, 1, 0, 0, 0, 520, 518, 1, 0, 0, 0, 521, 522, 3, 60, 30, 0, 522, 523, 5, 89, 0, 0, 523, 524, 3, 60, 30, 0, 524, 87, 1, 0, 0, 0, 525, 526, 5, 1, 0, 0, 526, 527, 3, 20, 10, 0, 527, 529, 3, 106, 53, 0, 528, 530, 3, 94, 47, 0, 529, 528, 1, 0, 0, 0, 529, 530, 1, 0, 0, 0, 530, 89, 1, 0, 0, 0, 531, 532, 5, 7, 0, 0, 532, 533, 3, 20, 10, 0, 533, 534, 3, 106, 53, 0, 534, 91, 1, 0, 0, 0, 535, 536, 5, 10, 0, 0, 536, 537, 3, 58, 29, 0, 537, 93, 1, 0, 0, 0, 538, 543, 3, 96, 48, 0, 539, 540, 5, 39, 0, 0, 540, 542, 3, 96, 48, 0, 541, 539, 1, 0, 0, 0, 542, 545, 1, 0, 0, 0, 543, 541, 1, 0, 0, 0, 543, 544, 1, 0, 0, 0, 544, 95, 1, 0, 0, 0, 545, 543, 1, 0, 0, 0, 546, 547, 3, 64, 32, 0, 547, 548, 5, 36, 0, 0, 548, 549, 3, 68, 34, 0, 549, 97, 1, 0, 0, 0, 550, 551, 7, 6, 0, 0, 551, 99, 1, 0, 0, 0, 552, 555, 3, 102, 51, 0, 553, 555, 3, 104, 52, 0, 554, 552, 1, 0, 0, 0, 554, 553, 1, 0, 0, 0, 555, 101, 1, 0, 0, 0, 556, 558, 7, 0, 0, 0, 557, 556, 1, 0, 0, 0, 557, 558, 1, 0, 0, 0, 558, 559, 1, 0, 0, 0, 559, 560, 5, 32, 0, 0, 560, 103, 1, 0, 0, 0, 561, 563, 7, 0, 0, 0, 562, 561, 1, 0, 0, 0, 562, 563, 1, 0, 0, 0, 563, 564, 1, 0, 0, 0, 564, 565, 5, 31, 0, 0, 565, 105, 1, 0, 0, 0, 566, 567, 5, 30, 0, 0, 567, 107, 1, 0, 0, 0, 568, 569, 7, 7, 0, 0, 569, 109, 1, 0, 0, 0, 570, 571, 5, 5, 0, 0, 571, 572, 3, 112, 56, 0, 572, 111, 1, 0, 0, 0, 573, 574, 5, 70, 0, 0, 574, 575, 3, 2, 1, 0, 575, 576, 5, 71, 0, 0, 576, 113, 1, 0, 0, 0, 577, 578, 5, 13, 0, 0, 578, 579, 5, 105, 0, 0, 579, 115, 1, 0, 0, 0, 580, 581, 5, 3, 0, 0, 581, 584, 5, 95, 0, 0, 582, 583, 5, 93, 0, 0, 583, 585, 3, 60, 30, 0, 584, 582, 1, 0, 0, 0, 584, 585, 1, 0, 0, 0, 585, 595, 1, 0, 0, 0, 586, 587, 5, 94, 0, 0, 587, 592, 3, 118, 59, 0, 588, 589, 5, 39, 0, 0, 589, 591, 3, 118, 59, 0, 590, 588, 1, 0, 0, 0, 591, 594, 1, 0, 0, 0, 592, 590, 1, 0, 0, 0, 592, 593, 1, 0, 0, 0, 593, 596, 1, 0, 0, 0, 594, 592, 1, 0, 0, 0, 595, 586, 1, 0, 0, 0, 595, 596, 1, 0, 0, 0, 596, 117, 1, 0, 0, 0, 597, 598, 3, 60, 30, 0, 598, 599, 5, 36, 0, 0, 599, 601, 1, 0, 0, 0, 600, 597, 1, 0, 0, 0, 600, 601, 1, 0, 0, 0, 601, 602, 1, 0, 0, 0, 602, 603, 3, 60, 30, 0, 603, 119, 1, 0, 0, 0, 604, 605, 5, 18, 0, 0, 605, 606, 3, 36, 18, 0, 606, 607, 5, 93, 0, 0, 607, 608, 3, 62, 31, 0, 608, 121, 1, 0, 0, 0, 609, 610, 5, 17, 0, 0, 610, 613, 3, 54, 27, 0, 611, 612, 5, 33, 0, 0, 612, 614, 3, 30, 15, 0, 613, 611, 1, 0, 0, 0, 613, 614, 1, 0, 0, 0, 614, 123, 1, 0, 0, 0, 615, 617, 7, 8, 0, 0, 616, 615, 1, 0, 0, 0, 616, 617, 1, 0, 0, 0, 617, 618, 1, 0, 0, 0, 618, 619, 5, 20, 0, 0, 619, 620, 3, 126, 63, 0, 620, 621, 3, 128, 64, 0, 621, 125, 1, 0, 0, 0, 622, 625, 3, 64, 32, 0, 623, 624, 5, 89, 0, 0, 624, 626, 3, 64, 32, 0, 625, 623, 1, 0, 0, 0, 625, 626, 1, 0, 0, 0, 626, 127, 1, 0, 0, 0, 627, 628, 5, 93, 0, 0, 628, 633, 3, 130, 65, 0, 629, 630, 5, 39, 0, 0, 630, 632, 3, 130, 65, 0, 631, 629, 1, 0, 0, 0, 632, 635, 1, 0, 0, 0, 633, 631, 1, 0, 0, 0, 633, 634, 1, 0, 0, 0, 634, 129, 1, 0, 0, 0, 635, 633, 1, 0, 0, 0, 636, 637, 3, 16, 8, 0, 637, 131, 1, 0, 0, 0, 62, 143, 152, 172, 184, 193, 201, 206, 214, 216, 221, 228, 233, 238, 248, 254, 262, 264, 275, 282, 293, 296, 312, 318, 328, 332, 337, 347, 355, 368, 372, 376, 383, 387, 394, 400, 407, 415, 423, 431, 448, 459, 470, 475, 479, 484, 495, 500, 504, 518, 529, 543, 554, 557, 562, 584, 592, 595, 600, 613, 616, 625, 633] \ No newline at end of file +[4, 1, 128, 627, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 138, 8, 1, 10, 1, 12, 1, 141, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 149, 8, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 169, 8, 3, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 181, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 188, 8, 5, 10, 5, 12, 5, 191, 9, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 198, 8, 5, 1, 5, 1, 5, 1, 5, 3, 5, 203, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 211, 8, 5, 10, 5, 12, 5, 214, 9, 5, 1, 6, 1, 6, 3, 6, 218, 8, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 3, 6, 225, 8, 6, 1, 6, 1, 6, 1, 6, 3, 6, 230, 8, 6, 1, 7, 1, 7, 1, 7, 3, 7, 235, 8, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 3, 8, 245, 8, 8, 1, 9, 1, 9, 1, 9, 1, 9, 3, 9, 251, 8, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 5, 9, 259, 8, 9, 10, 9, 12, 9, 262, 9, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 3, 10, 272, 8, 10, 1, 10, 1, 10, 1, 10, 5, 10, 277, 8, 10, 10, 10, 12, 10, 280, 9, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 5, 11, 288, 8, 11, 10, 11, 12, 11, 291, 9, 11, 3, 11, 293, 8, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 5, 15, 307, 8, 15, 10, 15, 12, 15, 310, 9, 15, 1, 16, 1, 16, 1, 16, 3, 16, 315, 8, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 5, 17, 323, 8, 17, 10, 17, 12, 17, 326, 9, 17, 1, 17, 3, 17, 329, 8, 17, 1, 18, 1, 18, 1, 18, 3, 18, 334, 8, 18, 1, 18, 1, 18, 1, 19, 1, 19, 1, 20, 1, 20, 1, 21, 1, 21, 1, 21, 1, 21, 5, 21, 346, 8, 21, 10, 21, 12, 21, 349, 9, 21, 1, 22, 1, 22, 1, 22, 1, 22, 5, 22, 355, 8, 22, 10, 22, 12, 22, 358, 9, 22, 1, 22, 3, 22, 361, 8, 22, 1, 22, 1, 22, 3, 22, 365, 8, 22, 1, 23, 1, 23, 1, 23, 1, 24, 1, 24, 3, 24, 372, 8, 24, 1, 24, 1, 24, 3, 24, 376, 8, 24, 1, 25, 1, 25, 1, 25, 5, 25, 381, 8, 25, 10, 25, 12, 25, 384, 9, 25, 1, 26, 1, 26, 1, 26, 3, 26, 389, 8, 26, 1, 27, 1, 27, 1, 27, 5, 27, 394, 8, 27, 10, 27, 12, 27, 397, 9, 27, 1, 28, 1, 28, 1, 28, 5, 28, 402, 8, 28, 10, 28, 12, 28, 405, 9, 28, 1, 29, 1, 29, 1, 29, 5, 29, 410, 8, 29, 10, 29, 12, 29, 413, 9, 29, 1, 30, 1, 30, 1, 31, 1, 31, 1, 31, 3, 31, 420, 8, 31, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 5, 32, 435, 8, 32, 10, 32, 12, 32, 438, 9, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 5, 32, 446, 8, 32, 10, 32, 12, 32, 449, 9, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 5, 32, 457, 8, 32, 10, 32, 12, 32, 460, 9, 32, 1, 32, 1, 32, 3, 32, 464, 8, 32, 1, 33, 1, 33, 3, 33, 468, 8, 33, 1, 34, 1, 34, 1, 34, 3, 34, 473, 8, 34, 1, 35, 1, 35, 1, 35, 1, 36, 1, 36, 1, 36, 1, 36, 5, 36, 482, 8, 36, 10, 36, 12, 36, 485, 9, 36, 1, 37, 1, 37, 3, 37, 489, 8, 37, 1, 37, 1, 37, 3, 37, 493, 8, 37, 1, 38, 1, 38, 1, 38, 1, 39, 1, 39, 1, 39, 1, 40, 1, 40, 1, 40, 1, 40, 5, 40, 505, 8, 40, 10, 40, 12, 40, 508, 9, 40, 1, 41, 1, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 42, 1, 42, 3, 42, 518, 8, 42, 1, 43, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 44, 1, 45, 1, 45, 1, 45, 5, 45, 530, 8, 45, 10, 45, 12, 45, 533, 9, 45, 1, 46, 1, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 48, 1, 48, 3, 48, 543, 8, 48, 1, 49, 3, 49, 546, 8, 49, 1, 49, 1, 49, 1, 50, 3, 50, 551, 8, 50, 1, 50, 1, 50, 1, 51, 1, 51, 1, 52, 1, 52, 1, 53, 1, 53, 1, 53, 1, 54, 1, 54, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 56, 1, 56, 3, 56, 573, 8, 56, 1, 56, 1, 56, 1, 56, 1, 56, 5, 56, 579, 8, 56, 10, 56, 12, 56, 582, 9, 56, 3, 56, 584, 8, 56, 1, 57, 1, 57, 1, 57, 3, 57, 589, 8, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 58, 1, 58, 1, 59, 1, 59, 1, 59, 1, 59, 3, 59, 602, 8, 59, 1, 60, 3, 60, 605, 8, 60, 1, 60, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 3, 61, 614, 8, 61, 1, 62, 1, 62, 1, 62, 1, 62, 5, 62, 620, 8, 62, 10, 62, 12, 62, 623, 9, 62, 1, 63, 1, 63, 1, 63, 0, 4, 2, 10, 18, 20, 64, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126, 0, 9, 1, 0, 64, 65, 1, 0, 66, 68, 2, 0, 30, 30, 81, 81, 1, 0, 72, 73, 2, 0, 35, 35, 40, 40, 2, 0, 43, 43, 46, 46, 2, 0, 42, 42, 56, 56, 2, 0, 57, 57, 59, 63, 1, 0, 22, 24, 654, 0, 128, 1, 0, 0, 0, 2, 131, 1, 0, 0, 0, 4, 148, 1, 0, 0, 0, 6, 168, 1, 0, 0, 0, 8, 170, 1, 0, 0, 0, 10, 202, 1, 0, 0, 0, 12, 229, 1, 0, 0, 0, 14, 231, 1, 0, 0, 0, 16, 244, 1, 0, 0, 0, 18, 250, 1, 0, 0, 0, 20, 271, 1, 0, 0, 0, 22, 281, 1, 0, 0, 0, 24, 296, 1, 0, 0, 0, 26, 298, 1, 0, 0, 0, 28, 300, 1, 0, 0, 0, 30, 303, 1, 0, 0, 0, 32, 314, 1, 0, 0, 0, 34, 318, 1, 0, 0, 0, 36, 333, 1, 0, 0, 0, 38, 337, 1, 0, 0, 0, 40, 339, 1, 0, 0, 0, 42, 341, 1, 0, 0, 0, 44, 350, 1, 0, 0, 0, 46, 366, 1, 0, 0, 0, 48, 369, 1, 0, 0, 0, 50, 377, 1, 0, 0, 0, 52, 385, 1, 0, 0, 0, 54, 390, 1, 0, 0, 0, 56, 398, 1, 0, 0, 0, 58, 406, 1, 0, 0, 0, 60, 414, 1, 0, 0, 0, 62, 419, 1, 0, 0, 0, 64, 463, 1, 0, 0, 0, 66, 467, 1, 0, 0, 0, 68, 472, 1, 0, 0, 0, 70, 474, 1, 0, 0, 0, 72, 477, 1, 0, 0, 0, 74, 486, 1, 0, 0, 0, 76, 494, 1, 0, 0, 0, 78, 497, 1, 0, 0, 0, 80, 500, 1, 0, 0, 0, 82, 509, 1, 0, 0, 0, 84, 513, 1, 0, 0, 0, 86, 519, 1, 0, 0, 0, 88, 523, 1, 0, 0, 0, 90, 526, 1, 0, 0, 0, 92, 534, 1, 0, 0, 0, 94, 538, 1, 0, 0, 0, 96, 542, 1, 0, 0, 0, 98, 545, 1, 0, 0, 0, 100, 550, 1, 0, 0, 0, 102, 554, 1, 0, 0, 0, 104, 556, 1, 0, 0, 0, 106, 558, 1, 0, 0, 0, 108, 561, 1, 0, 0, 0, 110, 565, 1, 0, 0, 0, 112, 568, 1, 0, 0, 0, 114, 588, 1, 0, 0, 0, 116, 592, 1, 0, 0, 0, 118, 597, 1, 0, 0, 0, 120, 604, 1, 0, 0, 0, 122, 610, 1, 0, 0, 0, 124, 615, 1, 0, 0, 0, 126, 624, 1, 0, 0, 0, 128, 129, 3, 2, 1, 0, 129, 130, 5, 0, 0, 1, 130, 1, 1, 0, 0, 0, 131, 132, 6, 1, -1, 0, 132, 133, 3, 4, 2, 0, 133, 139, 1, 0, 0, 0, 134, 135, 10, 1, 0, 0, 135, 136, 5, 29, 0, 0, 136, 138, 3, 6, 3, 0, 137, 134, 1, 0, 0, 0, 138, 141, 1, 0, 0, 0, 139, 137, 1, 0, 0, 0, 139, 140, 1, 0, 0, 0, 140, 3, 1, 0, 0, 0, 141, 139, 1, 0, 0, 0, 142, 149, 3, 106, 53, 0, 143, 149, 3, 34, 17, 0, 144, 149, 3, 28, 14, 0, 145, 149, 3, 110, 55, 0, 146, 147, 4, 2, 1, 0, 147, 149, 3, 44, 22, 0, 148, 142, 1, 0, 0, 0, 148, 143, 1, 0, 0, 0, 148, 144, 1, 0, 0, 0, 148, 145, 1, 0, 0, 0, 148, 146, 1, 0, 0, 0, 149, 5, 1, 0, 0, 0, 150, 169, 3, 46, 23, 0, 151, 169, 3, 8, 4, 0, 152, 169, 3, 76, 38, 0, 153, 169, 3, 70, 35, 0, 154, 169, 3, 48, 24, 0, 155, 169, 3, 72, 36, 0, 156, 169, 3, 78, 39, 0, 157, 169, 3, 80, 40, 0, 158, 169, 3, 84, 42, 0, 159, 169, 3, 86, 43, 0, 160, 169, 3, 112, 56, 0, 161, 169, 3, 88, 44, 0, 162, 163, 4, 3, 2, 0, 163, 169, 3, 118, 59, 0, 164, 165, 4, 3, 3, 0, 165, 169, 3, 116, 58, 0, 166, 167, 4, 3, 4, 0, 167, 169, 3, 120, 60, 0, 168, 150, 1, 0, 0, 0, 168, 151, 1, 0, 0, 0, 168, 152, 1, 0, 0, 0, 168, 153, 1, 0, 0, 0, 168, 154, 1, 0, 0, 0, 168, 155, 1, 0, 0, 0, 168, 156, 1, 0, 0, 0, 168, 157, 1, 0, 0, 0, 168, 158, 1, 0, 0, 0, 168, 159, 1, 0, 0, 0, 168, 160, 1, 0, 0, 0, 168, 161, 1, 0, 0, 0, 168, 162, 1, 0, 0, 0, 168, 164, 1, 0, 0, 0, 168, 166, 1, 0, 0, 0, 169, 7, 1, 0, 0, 0, 170, 171, 5, 16, 0, 0, 171, 172, 3, 10, 5, 0, 172, 9, 1, 0, 0, 0, 173, 174, 6, 5, -1, 0, 174, 175, 5, 49, 0, 0, 175, 203, 3, 10, 5, 8, 176, 203, 3, 16, 8, 0, 177, 203, 3, 12, 6, 0, 178, 180, 3, 16, 8, 0, 179, 181, 5, 49, 0, 0, 180, 179, 1, 0, 0, 0, 180, 181, 1, 0, 0, 0, 181, 182, 1, 0, 0, 0, 182, 183, 5, 44, 0, 0, 183, 184, 5, 48, 0, 0, 184, 189, 3, 16, 8, 0, 185, 186, 5, 39, 0, 0, 186, 188, 3, 16, 8, 0, 187, 185, 1, 0, 0, 0, 188, 191, 1, 0, 0, 0, 189, 187, 1, 0, 0, 0, 189, 190, 1, 0, 0, 0, 190, 192, 1, 0, 0, 0, 191, 189, 1, 0, 0, 0, 192, 193, 5, 55, 0, 0, 193, 203, 1, 0, 0, 0, 194, 195, 3, 16, 8, 0, 195, 197, 5, 45, 0, 0, 196, 198, 5, 49, 0, 0, 197, 196, 1, 0, 0, 0, 197, 198, 1, 0, 0, 0, 198, 199, 1, 0, 0, 0, 199, 200, 5, 50, 0, 0, 200, 203, 1, 0, 0, 0, 201, 203, 3, 14, 7, 0, 202, 173, 1, 0, 0, 0, 202, 176, 1, 0, 0, 0, 202, 177, 1, 0, 0, 0, 202, 178, 1, 0, 0, 0, 202, 194, 1, 0, 0, 0, 202, 201, 1, 0, 0, 0, 203, 212, 1, 0, 0, 0, 204, 205, 10, 5, 0, 0, 205, 206, 5, 34, 0, 0, 206, 211, 3, 10, 5, 6, 207, 208, 10, 4, 0, 0, 208, 209, 5, 52, 0, 0, 209, 211, 3, 10, 5, 5, 210, 204, 1, 0, 0, 0, 210, 207, 1, 0, 0, 0, 211, 214, 1, 0, 0, 0, 212, 210, 1, 0, 0, 0, 212, 213, 1, 0, 0, 0, 213, 11, 1, 0, 0, 0, 214, 212, 1, 0, 0, 0, 215, 217, 3, 16, 8, 0, 216, 218, 5, 49, 0, 0, 217, 216, 1, 0, 0, 0, 217, 218, 1, 0, 0, 0, 218, 219, 1, 0, 0, 0, 219, 220, 5, 47, 0, 0, 220, 221, 3, 102, 51, 0, 221, 230, 1, 0, 0, 0, 222, 224, 3, 16, 8, 0, 223, 225, 5, 49, 0, 0, 224, 223, 1, 0, 0, 0, 224, 225, 1, 0, 0, 0, 225, 226, 1, 0, 0, 0, 226, 227, 5, 54, 0, 0, 227, 228, 3, 102, 51, 0, 228, 230, 1, 0, 0, 0, 229, 215, 1, 0, 0, 0, 229, 222, 1, 0, 0, 0, 230, 13, 1, 0, 0, 0, 231, 234, 3, 54, 27, 0, 232, 233, 5, 37, 0, 0, 233, 235, 3, 26, 13, 0, 234, 232, 1, 0, 0, 0, 234, 235, 1, 0, 0, 0, 235, 236, 1, 0, 0, 0, 236, 237, 5, 38, 0, 0, 237, 238, 3, 64, 32, 0, 238, 15, 1, 0, 0, 0, 239, 245, 3, 18, 9, 0, 240, 241, 3, 18, 9, 0, 241, 242, 3, 104, 52, 0, 242, 243, 3, 18, 9, 0, 243, 245, 1, 0, 0, 0, 244, 239, 1, 0, 0, 0, 244, 240, 1, 0, 0, 0, 245, 17, 1, 0, 0, 0, 246, 247, 6, 9, -1, 0, 247, 251, 3, 20, 10, 0, 248, 249, 7, 0, 0, 0, 249, 251, 3, 18, 9, 3, 250, 246, 1, 0, 0, 0, 250, 248, 1, 0, 0, 0, 251, 260, 1, 0, 0, 0, 252, 253, 10, 2, 0, 0, 253, 254, 7, 1, 0, 0, 254, 259, 3, 18, 9, 3, 255, 256, 10, 1, 0, 0, 256, 257, 7, 0, 0, 0, 257, 259, 3, 18, 9, 2, 258, 252, 1, 0, 0, 0, 258, 255, 1, 0, 0, 0, 259, 262, 1, 0, 0, 0, 260, 258, 1, 0, 0, 0, 260, 261, 1, 0, 0, 0, 261, 19, 1, 0, 0, 0, 262, 260, 1, 0, 0, 0, 263, 264, 6, 10, -1, 0, 264, 272, 3, 64, 32, 0, 265, 272, 3, 54, 27, 0, 266, 272, 3, 22, 11, 0, 267, 268, 5, 48, 0, 0, 268, 269, 3, 10, 5, 0, 269, 270, 5, 55, 0, 0, 270, 272, 1, 0, 0, 0, 271, 263, 1, 0, 0, 0, 271, 265, 1, 0, 0, 0, 271, 266, 1, 0, 0, 0, 271, 267, 1, 0, 0, 0, 272, 278, 1, 0, 0, 0, 273, 274, 10, 1, 0, 0, 274, 275, 5, 37, 0, 0, 275, 277, 3, 26, 13, 0, 276, 273, 1, 0, 0, 0, 277, 280, 1, 0, 0, 0, 278, 276, 1, 0, 0, 0, 278, 279, 1, 0, 0, 0, 279, 21, 1, 0, 0, 0, 280, 278, 1, 0, 0, 0, 281, 282, 3, 24, 12, 0, 282, 292, 5, 48, 0, 0, 283, 293, 5, 66, 0, 0, 284, 289, 3, 10, 5, 0, 285, 286, 5, 39, 0, 0, 286, 288, 3, 10, 5, 0, 287, 285, 1, 0, 0, 0, 288, 291, 1, 0, 0, 0, 289, 287, 1, 0, 0, 0, 289, 290, 1, 0, 0, 0, 290, 293, 1, 0, 0, 0, 291, 289, 1, 0, 0, 0, 292, 283, 1, 0, 0, 0, 292, 284, 1, 0, 0, 0, 292, 293, 1, 0, 0, 0, 293, 294, 1, 0, 0, 0, 294, 295, 5, 55, 0, 0, 295, 23, 1, 0, 0, 0, 296, 297, 3, 68, 34, 0, 297, 25, 1, 0, 0, 0, 298, 299, 3, 60, 30, 0, 299, 27, 1, 0, 0, 0, 300, 301, 5, 12, 0, 0, 301, 302, 3, 30, 15, 0, 302, 29, 1, 0, 0, 0, 303, 308, 3, 32, 16, 0, 304, 305, 5, 39, 0, 0, 305, 307, 3, 32, 16, 0, 306, 304, 1, 0, 0, 0, 307, 310, 1, 0, 0, 0, 308, 306, 1, 0, 0, 0, 308, 309, 1, 0, 0, 0, 309, 31, 1, 0, 0, 0, 310, 308, 1, 0, 0, 0, 311, 312, 3, 54, 27, 0, 312, 313, 5, 36, 0, 0, 313, 315, 1, 0, 0, 0, 314, 311, 1, 0, 0, 0, 314, 315, 1, 0, 0, 0, 315, 316, 1, 0, 0, 0, 316, 317, 3, 10, 5, 0, 317, 33, 1, 0, 0, 0, 318, 319, 5, 6, 0, 0, 319, 324, 3, 36, 18, 0, 320, 321, 5, 39, 0, 0, 321, 323, 3, 36, 18, 0, 322, 320, 1, 0, 0, 0, 323, 326, 1, 0, 0, 0, 324, 322, 1, 0, 0, 0, 324, 325, 1, 0, 0, 0, 325, 328, 1, 0, 0, 0, 326, 324, 1, 0, 0, 0, 327, 329, 3, 42, 21, 0, 328, 327, 1, 0, 0, 0, 328, 329, 1, 0, 0, 0, 329, 35, 1, 0, 0, 0, 330, 331, 3, 38, 19, 0, 331, 332, 5, 38, 0, 0, 332, 334, 1, 0, 0, 0, 333, 330, 1, 0, 0, 0, 333, 334, 1, 0, 0, 0, 334, 335, 1, 0, 0, 0, 335, 336, 3, 40, 20, 0, 336, 37, 1, 0, 0, 0, 337, 338, 5, 81, 0, 0, 338, 39, 1, 0, 0, 0, 339, 340, 7, 2, 0, 0, 340, 41, 1, 0, 0, 0, 341, 342, 5, 80, 0, 0, 342, 347, 5, 81, 0, 0, 343, 344, 5, 39, 0, 0, 344, 346, 5, 81, 0, 0, 345, 343, 1, 0, 0, 0, 346, 349, 1, 0, 0, 0, 347, 345, 1, 0, 0, 0, 347, 348, 1, 0, 0, 0, 348, 43, 1, 0, 0, 0, 349, 347, 1, 0, 0, 0, 350, 351, 5, 19, 0, 0, 351, 356, 3, 36, 18, 0, 352, 353, 5, 39, 0, 0, 353, 355, 3, 36, 18, 0, 354, 352, 1, 0, 0, 0, 355, 358, 1, 0, 0, 0, 356, 354, 1, 0, 0, 0, 356, 357, 1, 0, 0, 0, 357, 360, 1, 0, 0, 0, 358, 356, 1, 0, 0, 0, 359, 361, 3, 50, 25, 0, 360, 359, 1, 0, 0, 0, 360, 361, 1, 0, 0, 0, 361, 364, 1, 0, 0, 0, 362, 363, 5, 33, 0, 0, 363, 365, 3, 30, 15, 0, 364, 362, 1, 0, 0, 0, 364, 365, 1, 0, 0, 0, 365, 45, 1, 0, 0, 0, 366, 367, 5, 4, 0, 0, 367, 368, 3, 30, 15, 0, 368, 47, 1, 0, 0, 0, 369, 371, 5, 15, 0, 0, 370, 372, 3, 50, 25, 0, 371, 370, 1, 0, 0, 0, 371, 372, 1, 0, 0, 0, 372, 375, 1, 0, 0, 0, 373, 374, 5, 33, 0, 0, 374, 376, 3, 30, 15, 0, 375, 373, 1, 0, 0, 0, 375, 376, 1, 0, 0, 0, 376, 49, 1, 0, 0, 0, 377, 382, 3, 52, 26, 0, 378, 379, 5, 39, 0, 0, 379, 381, 3, 52, 26, 0, 380, 378, 1, 0, 0, 0, 381, 384, 1, 0, 0, 0, 382, 380, 1, 0, 0, 0, 382, 383, 1, 0, 0, 0, 383, 51, 1, 0, 0, 0, 384, 382, 1, 0, 0, 0, 385, 388, 3, 32, 16, 0, 386, 387, 5, 16, 0, 0, 387, 389, 3, 10, 5, 0, 388, 386, 1, 0, 0, 0, 388, 389, 1, 0, 0, 0, 389, 53, 1, 0, 0, 0, 390, 395, 3, 68, 34, 0, 391, 392, 5, 41, 0, 0, 392, 394, 3, 68, 34, 0, 393, 391, 1, 0, 0, 0, 394, 397, 1, 0, 0, 0, 395, 393, 1, 0, 0, 0, 395, 396, 1, 0, 0, 0, 396, 55, 1, 0, 0, 0, 397, 395, 1, 0, 0, 0, 398, 403, 3, 62, 31, 0, 399, 400, 5, 41, 0, 0, 400, 402, 3, 62, 31, 0, 401, 399, 1, 0, 0, 0, 402, 405, 1, 0, 0, 0, 403, 401, 1, 0, 0, 0, 403, 404, 1, 0, 0, 0, 404, 57, 1, 0, 0, 0, 405, 403, 1, 0, 0, 0, 406, 411, 3, 56, 28, 0, 407, 408, 5, 39, 0, 0, 408, 410, 3, 56, 28, 0, 409, 407, 1, 0, 0, 0, 410, 413, 1, 0, 0, 0, 411, 409, 1, 0, 0, 0, 411, 412, 1, 0, 0, 0, 412, 59, 1, 0, 0, 0, 413, 411, 1, 0, 0, 0, 414, 415, 7, 3, 0, 0, 415, 61, 1, 0, 0, 0, 416, 420, 5, 85, 0, 0, 417, 418, 4, 31, 10, 0, 418, 420, 3, 66, 33, 0, 419, 416, 1, 0, 0, 0, 419, 417, 1, 0, 0, 0, 420, 63, 1, 0, 0, 0, 421, 464, 5, 50, 0, 0, 422, 423, 3, 100, 50, 0, 423, 424, 5, 72, 0, 0, 424, 464, 1, 0, 0, 0, 425, 464, 3, 98, 49, 0, 426, 464, 3, 100, 50, 0, 427, 464, 3, 94, 47, 0, 428, 464, 3, 66, 33, 0, 429, 464, 3, 102, 51, 0, 430, 431, 5, 70, 0, 0, 431, 436, 3, 96, 48, 0, 432, 433, 5, 39, 0, 0, 433, 435, 3, 96, 48, 0, 434, 432, 1, 0, 0, 0, 435, 438, 1, 0, 0, 0, 436, 434, 1, 0, 0, 0, 436, 437, 1, 0, 0, 0, 437, 439, 1, 0, 0, 0, 438, 436, 1, 0, 0, 0, 439, 440, 5, 71, 0, 0, 440, 464, 1, 0, 0, 0, 441, 442, 5, 70, 0, 0, 442, 447, 3, 94, 47, 0, 443, 444, 5, 39, 0, 0, 444, 446, 3, 94, 47, 0, 445, 443, 1, 0, 0, 0, 446, 449, 1, 0, 0, 0, 447, 445, 1, 0, 0, 0, 447, 448, 1, 0, 0, 0, 448, 450, 1, 0, 0, 0, 449, 447, 1, 0, 0, 0, 450, 451, 5, 71, 0, 0, 451, 464, 1, 0, 0, 0, 452, 453, 5, 70, 0, 0, 453, 458, 3, 102, 51, 0, 454, 455, 5, 39, 0, 0, 455, 457, 3, 102, 51, 0, 456, 454, 1, 0, 0, 0, 457, 460, 1, 0, 0, 0, 458, 456, 1, 0, 0, 0, 458, 459, 1, 0, 0, 0, 459, 461, 1, 0, 0, 0, 460, 458, 1, 0, 0, 0, 461, 462, 5, 71, 0, 0, 462, 464, 1, 0, 0, 0, 463, 421, 1, 0, 0, 0, 463, 422, 1, 0, 0, 0, 463, 425, 1, 0, 0, 0, 463, 426, 1, 0, 0, 0, 463, 427, 1, 0, 0, 0, 463, 428, 1, 0, 0, 0, 463, 429, 1, 0, 0, 0, 463, 430, 1, 0, 0, 0, 463, 441, 1, 0, 0, 0, 463, 452, 1, 0, 0, 0, 464, 65, 1, 0, 0, 0, 465, 468, 5, 53, 0, 0, 466, 468, 5, 69, 0, 0, 467, 465, 1, 0, 0, 0, 467, 466, 1, 0, 0, 0, 468, 67, 1, 0, 0, 0, 469, 473, 3, 60, 30, 0, 470, 471, 4, 34, 11, 0, 471, 473, 3, 66, 33, 0, 472, 469, 1, 0, 0, 0, 472, 470, 1, 0, 0, 0, 473, 69, 1, 0, 0, 0, 474, 475, 5, 9, 0, 0, 475, 476, 5, 31, 0, 0, 476, 71, 1, 0, 0, 0, 477, 478, 5, 14, 0, 0, 478, 483, 3, 74, 37, 0, 479, 480, 5, 39, 0, 0, 480, 482, 3, 74, 37, 0, 481, 479, 1, 0, 0, 0, 482, 485, 1, 0, 0, 0, 483, 481, 1, 0, 0, 0, 483, 484, 1, 0, 0, 0, 484, 73, 1, 0, 0, 0, 485, 483, 1, 0, 0, 0, 486, 488, 3, 10, 5, 0, 487, 489, 7, 4, 0, 0, 488, 487, 1, 0, 0, 0, 488, 489, 1, 0, 0, 0, 489, 492, 1, 0, 0, 0, 490, 491, 5, 51, 0, 0, 491, 493, 7, 5, 0, 0, 492, 490, 1, 0, 0, 0, 492, 493, 1, 0, 0, 0, 493, 75, 1, 0, 0, 0, 494, 495, 5, 8, 0, 0, 495, 496, 3, 58, 29, 0, 496, 77, 1, 0, 0, 0, 497, 498, 5, 2, 0, 0, 498, 499, 3, 58, 29, 0, 499, 79, 1, 0, 0, 0, 500, 501, 5, 11, 0, 0, 501, 506, 3, 82, 41, 0, 502, 503, 5, 39, 0, 0, 503, 505, 3, 82, 41, 0, 504, 502, 1, 0, 0, 0, 505, 508, 1, 0, 0, 0, 506, 504, 1, 0, 0, 0, 506, 507, 1, 0, 0, 0, 507, 81, 1, 0, 0, 0, 508, 506, 1, 0, 0, 0, 509, 510, 3, 56, 28, 0, 510, 511, 5, 89, 0, 0, 511, 512, 3, 56, 28, 0, 512, 83, 1, 0, 0, 0, 513, 514, 5, 1, 0, 0, 514, 515, 3, 20, 10, 0, 515, 517, 3, 102, 51, 0, 516, 518, 3, 90, 45, 0, 517, 516, 1, 0, 0, 0, 517, 518, 1, 0, 0, 0, 518, 85, 1, 0, 0, 0, 519, 520, 5, 7, 0, 0, 520, 521, 3, 20, 10, 0, 521, 522, 3, 102, 51, 0, 522, 87, 1, 0, 0, 0, 523, 524, 5, 10, 0, 0, 524, 525, 3, 54, 27, 0, 525, 89, 1, 0, 0, 0, 526, 531, 3, 92, 46, 0, 527, 528, 5, 39, 0, 0, 528, 530, 3, 92, 46, 0, 529, 527, 1, 0, 0, 0, 530, 533, 1, 0, 0, 0, 531, 529, 1, 0, 0, 0, 531, 532, 1, 0, 0, 0, 532, 91, 1, 0, 0, 0, 533, 531, 1, 0, 0, 0, 534, 535, 3, 60, 30, 0, 535, 536, 5, 36, 0, 0, 536, 537, 3, 64, 32, 0, 537, 93, 1, 0, 0, 0, 538, 539, 7, 6, 0, 0, 539, 95, 1, 0, 0, 0, 540, 543, 3, 98, 49, 0, 541, 543, 3, 100, 50, 0, 542, 540, 1, 0, 0, 0, 542, 541, 1, 0, 0, 0, 543, 97, 1, 0, 0, 0, 544, 546, 7, 0, 0, 0, 545, 544, 1, 0, 0, 0, 545, 546, 1, 0, 0, 0, 546, 547, 1, 0, 0, 0, 547, 548, 5, 32, 0, 0, 548, 99, 1, 0, 0, 0, 549, 551, 7, 0, 0, 0, 550, 549, 1, 0, 0, 0, 550, 551, 1, 0, 0, 0, 551, 552, 1, 0, 0, 0, 552, 553, 5, 31, 0, 0, 553, 101, 1, 0, 0, 0, 554, 555, 5, 30, 0, 0, 555, 103, 1, 0, 0, 0, 556, 557, 7, 7, 0, 0, 557, 105, 1, 0, 0, 0, 558, 559, 5, 5, 0, 0, 559, 560, 3, 108, 54, 0, 560, 107, 1, 0, 0, 0, 561, 562, 5, 70, 0, 0, 562, 563, 3, 2, 1, 0, 563, 564, 5, 71, 0, 0, 564, 109, 1, 0, 0, 0, 565, 566, 5, 13, 0, 0, 566, 567, 5, 105, 0, 0, 567, 111, 1, 0, 0, 0, 568, 569, 5, 3, 0, 0, 569, 572, 5, 95, 0, 0, 570, 571, 5, 93, 0, 0, 571, 573, 3, 56, 28, 0, 572, 570, 1, 0, 0, 0, 572, 573, 1, 0, 0, 0, 573, 583, 1, 0, 0, 0, 574, 575, 5, 94, 0, 0, 575, 580, 3, 114, 57, 0, 576, 577, 5, 39, 0, 0, 577, 579, 3, 114, 57, 0, 578, 576, 1, 0, 0, 0, 579, 582, 1, 0, 0, 0, 580, 578, 1, 0, 0, 0, 580, 581, 1, 0, 0, 0, 581, 584, 1, 0, 0, 0, 582, 580, 1, 0, 0, 0, 583, 574, 1, 0, 0, 0, 583, 584, 1, 0, 0, 0, 584, 113, 1, 0, 0, 0, 585, 586, 3, 56, 28, 0, 586, 587, 5, 36, 0, 0, 587, 589, 1, 0, 0, 0, 588, 585, 1, 0, 0, 0, 588, 589, 1, 0, 0, 0, 589, 590, 1, 0, 0, 0, 590, 591, 3, 56, 28, 0, 591, 115, 1, 0, 0, 0, 592, 593, 5, 18, 0, 0, 593, 594, 3, 36, 18, 0, 594, 595, 5, 93, 0, 0, 595, 596, 3, 58, 29, 0, 596, 117, 1, 0, 0, 0, 597, 598, 5, 17, 0, 0, 598, 601, 3, 50, 25, 0, 599, 600, 5, 33, 0, 0, 600, 602, 3, 30, 15, 0, 601, 599, 1, 0, 0, 0, 601, 602, 1, 0, 0, 0, 602, 119, 1, 0, 0, 0, 603, 605, 7, 8, 0, 0, 604, 603, 1, 0, 0, 0, 604, 605, 1, 0, 0, 0, 605, 606, 1, 0, 0, 0, 606, 607, 5, 20, 0, 0, 607, 608, 3, 122, 61, 0, 608, 609, 3, 124, 62, 0, 609, 121, 1, 0, 0, 0, 610, 613, 3, 60, 30, 0, 611, 612, 5, 89, 0, 0, 612, 614, 3, 60, 30, 0, 613, 611, 1, 0, 0, 0, 613, 614, 1, 0, 0, 0, 614, 123, 1, 0, 0, 0, 615, 616, 5, 93, 0, 0, 616, 621, 3, 126, 63, 0, 617, 618, 5, 39, 0, 0, 618, 620, 3, 126, 63, 0, 619, 617, 1, 0, 0, 0, 620, 623, 1, 0, 0, 0, 621, 619, 1, 0, 0, 0, 621, 622, 1, 0, 0, 0, 622, 125, 1, 0, 0, 0, 623, 621, 1, 0, 0, 0, 624, 625, 3, 16, 8, 0, 625, 127, 1, 0, 0, 0, 61, 139, 148, 168, 180, 189, 197, 202, 210, 212, 217, 224, 229, 234, 244, 250, 258, 260, 271, 278, 289, 292, 308, 314, 324, 328, 333, 347, 356, 360, 364, 371, 375, 382, 388, 395, 403, 411, 419, 436, 447, 458, 463, 467, 472, 483, 488, 492, 506, 517, 531, 542, 545, 550, 572, 580, 583, 588, 601, 604, 613, 621] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java index a56035364641d..e272dc7f477a4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java @@ -59,35 +59,34 @@ public class EsqlBaseParser extends ParserConfig { RULE_primaryExpression = 10, RULE_functionExpression = 11, RULE_functionName = 12, RULE_dataType = 13, RULE_rowCommand = 14, RULE_fields = 15, RULE_field = 16, RULE_fromCommand = 17, RULE_indexPattern = 18, RULE_clusterString = 19, - RULE_indexString = 20, RULE_metadata = 21, RULE_metadataOption = 22, RULE_deprecated_metadata = 23, - RULE_metricsCommand = 24, RULE_evalCommand = 25, RULE_statsCommand = 26, - RULE_aggFields = 27, RULE_aggField = 28, RULE_qualifiedName = 29, RULE_qualifiedNamePattern = 30, - RULE_qualifiedNamePatterns = 31, RULE_identifier = 32, RULE_identifierPattern = 33, - RULE_constant = 34, RULE_parameter = 35, RULE_identifierOrParameter = 36, - RULE_limitCommand = 37, RULE_sortCommand = 38, RULE_orderExpression = 39, - RULE_keepCommand = 40, RULE_dropCommand = 41, RULE_renameCommand = 42, - RULE_renameClause = 43, RULE_dissectCommand = 44, RULE_grokCommand = 45, - RULE_mvExpandCommand = 46, RULE_commandOptions = 47, RULE_commandOption = 48, - RULE_booleanValue = 49, RULE_numericValue = 50, RULE_decimalValue = 51, - RULE_integerValue = 52, RULE_string = 53, RULE_comparisonOperator = 54, - RULE_explainCommand = 55, RULE_subqueryExpression = 56, RULE_showCommand = 57, - RULE_enrichCommand = 58, RULE_enrichWithClause = 59, RULE_lookupCommand = 60, - RULE_inlinestatsCommand = 61, RULE_joinCommand = 62, RULE_joinTarget = 63, - RULE_joinCondition = 64, RULE_joinPredicate = 65; + RULE_indexString = 20, RULE_metadata = 21, RULE_metricsCommand = 22, RULE_evalCommand = 23, + RULE_statsCommand = 24, RULE_aggFields = 25, RULE_aggField = 26, RULE_qualifiedName = 27, + RULE_qualifiedNamePattern = 28, RULE_qualifiedNamePatterns = 29, RULE_identifier = 30, + RULE_identifierPattern = 31, RULE_constant = 32, RULE_parameter = 33, + RULE_identifierOrParameter = 34, RULE_limitCommand = 35, RULE_sortCommand = 36, + RULE_orderExpression = 37, RULE_keepCommand = 38, RULE_dropCommand = 39, + RULE_renameCommand = 40, RULE_renameClause = 41, RULE_dissectCommand = 42, + RULE_grokCommand = 43, RULE_mvExpandCommand = 44, RULE_commandOptions = 45, + RULE_commandOption = 46, RULE_booleanValue = 47, RULE_numericValue = 48, + RULE_decimalValue = 49, RULE_integerValue = 50, RULE_string = 51, RULE_comparisonOperator = 52, + RULE_explainCommand = 53, RULE_subqueryExpression = 54, RULE_showCommand = 55, + RULE_enrichCommand = 56, RULE_enrichWithClause = 57, RULE_lookupCommand = 58, + RULE_inlinestatsCommand = 59, RULE_joinCommand = 60, RULE_joinTarget = 61, + RULE_joinCondition = 62, RULE_joinPredicate = 63; private static String[] makeRuleNames() { return new String[] { "singleStatement", "query", "sourceCommand", "processingCommand", "whereCommand", "booleanExpression", "regexBooleanExpression", "matchBooleanExpression", "valueExpression", "operatorExpression", "primaryExpression", "functionExpression", "functionName", "dataType", "rowCommand", "fields", "field", "fromCommand", - "indexPattern", "clusterString", "indexString", "metadata", "metadataOption", - "deprecated_metadata", "metricsCommand", "evalCommand", "statsCommand", - "aggFields", "aggField", "qualifiedName", "qualifiedNamePattern", "qualifiedNamePatterns", - "identifier", "identifierPattern", "constant", "parameter", "identifierOrParameter", - "limitCommand", "sortCommand", "orderExpression", "keepCommand", "dropCommand", - "renameCommand", "renameClause", "dissectCommand", "grokCommand", "mvExpandCommand", - "commandOptions", "commandOption", "booleanValue", "numericValue", "decimalValue", - "integerValue", "string", "comparisonOperator", "explainCommand", "subqueryExpression", + "indexPattern", "clusterString", "indexString", "metadata", "metricsCommand", + "evalCommand", "statsCommand", "aggFields", "aggField", "qualifiedName", + "qualifiedNamePattern", "qualifiedNamePatterns", "identifier", "identifierPattern", + "constant", "parameter", "identifierOrParameter", "limitCommand", "sortCommand", + "orderExpression", "keepCommand", "dropCommand", "renameCommand", "renameClause", + "dissectCommand", "grokCommand", "mvExpandCommand", "commandOptions", + "commandOption", "booleanValue", "numericValue", "decimalValue", "integerValue", + "string", "comparisonOperator", "explainCommand", "subqueryExpression", "showCommand", "enrichCommand", "enrichWithClause", "lookupCommand", "inlinestatsCommand", "joinCommand", "joinTarget", "joinCondition", "joinPredicate" }; @@ -226,9 +225,9 @@ public final SingleStatementContext singleStatement() throws RecognitionExceptio try { enterOuterAlt(_localctx, 1); { - setState(132); + setState(128); query(0); - setState(133); + setState(129); match(EOF); } } @@ -324,11 +323,11 @@ private QueryContext query(int _p) throws RecognitionException { _ctx = _localctx; _prevctx = _localctx; - setState(136); + setState(132); sourceCommand(); } _ctx.stop = _input.LT(-1); - setState(143); + setState(139); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,0,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -339,16 +338,16 @@ private QueryContext query(int _p) throws RecognitionException { { _localctx = new CompositeQueryContext(new QueryContext(_parentctx, _parentState)); pushNewRecursionContext(_localctx, _startState, RULE_query); - setState(138); + setState(134); if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)"); - setState(139); + setState(135); match(PIPE); - setState(140); + setState(136); processingCommand(); } } } - setState(145); + setState(141); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,0,_ctx); } @@ -406,43 +405,43 @@ public final SourceCommandContext sourceCommand() throws RecognitionException { SourceCommandContext _localctx = new SourceCommandContext(_ctx, getState()); enterRule(_localctx, 4, RULE_sourceCommand); try { - setState(152); + setState(148); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,1,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(146); + setState(142); explainCommand(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(147); + setState(143); fromCommand(); } break; case 3: enterOuterAlt(_localctx, 3); { - setState(148); + setState(144); rowCommand(); } break; case 4: enterOuterAlt(_localctx, 4); { - setState(149); + setState(145); showCommand(); } break; case 5: enterOuterAlt(_localctx, 5); { - setState(150); + setState(146); if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(151); + setState(147); metricsCommand(); } break; @@ -530,117 +529,117 @@ public final ProcessingCommandContext processingCommand() throws RecognitionExce ProcessingCommandContext _localctx = new ProcessingCommandContext(_ctx, getState()); enterRule(_localctx, 6, RULE_processingCommand); try { - setState(172); + setState(168); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,2,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(154); + setState(150); evalCommand(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(155); + setState(151); whereCommand(); } break; case 3: enterOuterAlt(_localctx, 3); { - setState(156); + setState(152); keepCommand(); } break; case 4: enterOuterAlt(_localctx, 4); { - setState(157); + setState(153); limitCommand(); } break; case 5: enterOuterAlt(_localctx, 5); { - setState(158); + setState(154); statsCommand(); } break; case 6: enterOuterAlt(_localctx, 6); { - setState(159); + setState(155); sortCommand(); } break; case 7: enterOuterAlt(_localctx, 7); { - setState(160); + setState(156); dropCommand(); } break; case 8: enterOuterAlt(_localctx, 8); { - setState(161); + setState(157); renameCommand(); } break; case 9: enterOuterAlt(_localctx, 9); { - setState(162); + setState(158); dissectCommand(); } break; case 10: enterOuterAlt(_localctx, 10); { - setState(163); + setState(159); grokCommand(); } break; case 11: enterOuterAlt(_localctx, 11); { - setState(164); + setState(160); enrichCommand(); } break; case 12: enterOuterAlt(_localctx, 12); { - setState(165); + setState(161); mvExpandCommand(); } break; case 13: enterOuterAlt(_localctx, 13); { - setState(166); + setState(162); if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(167); + setState(163); inlinestatsCommand(); } break; case 14: enterOuterAlt(_localctx, 14); { - setState(168); + setState(164); if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(169); + setState(165); lookupCommand(); } break; case 15: enterOuterAlt(_localctx, 15); { - setState(170); + setState(166); if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(171); + setState(167); joinCommand(); } break; @@ -689,9 +688,9 @@ public final WhereCommandContext whereCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(174); + setState(170); match(WHERE); - setState(175); + setState(171); booleanExpression(0); } } @@ -907,7 +906,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc int _alt; enterOuterAlt(_localctx, 1); { - setState(206); + setState(202); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,6,_ctx) ) { case 1: @@ -916,9 +915,9 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _ctx = _localctx; _prevctx = _localctx; - setState(178); + setState(174); match(NOT); - setState(179); + setState(175); booleanExpression(8); } break; @@ -927,7 +926,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new BooleanDefaultContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(180); + setState(176); valueExpression(); } break; @@ -936,7 +935,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new RegexExpressionContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(181); + setState(177); regexBooleanExpression(); } break; @@ -945,41 +944,41 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new LogicalInContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(182); + setState(178); valueExpression(); - setState(184); + setState(180); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(183); + setState(179); match(NOT); } } - setState(186); + setState(182); match(IN); - setState(187); + setState(183); match(LP); - setState(188); + setState(184); valueExpression(); - setState(193); + setState(189); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(189); + setState(185); match(COMMA); - setState(190); + setState(186); valueExpression(); } } - setState(195); + setState(191); _errHandler.sync(this); _la = _input.LA(1); } - setState(196); + setState(192); match(RP); } break; @@ -988,21 +987,21 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new IsNullContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(198); + setState(194); valueExpression(); - setState(199); + setState(195); match(IS); - setState(201); + setState(197); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(200); + setState(196); match(NOT); } } - setState(203); + setState(199); match(NULL); } break; @@ -1011,13 +1010,13 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new MatchExpressionContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(205); + setState(201); matchBooleanExpression(); } break; } _ctx.stop = _input.LT(-1); - setState(216); + setState(212); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,8,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -1025,7 +1024,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc if ( _parseListeners!=null ) triggerExitRuleEvent(); _prevctx = _localctx; { - setState(214); + setState(210); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,7,_ctx) ) { case 1: @@ -1033,11 +1032,11 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new LogicalBinaryContext(new BooleanExpressionContext(_parentctx, _parentState)); ((LogicalBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_booleanExpression); - setState(208); + setState(204); if (!(precpred(_ctx, 5))) throw new FailedPredicateException(this, "precpred(_ctx, 5)"); - setState(209); + setState(205); ((LogicalBinaryContext)_localctx).operator = match(AND); - setState(210); + setState(206); ((LogicalBinaryContext)_localctx).right = booleanExpression(6); } break; @@ -1046,18 +1045,18 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new LogicalBinaryContext(new BooleanExpressionContext(_parentctx, _parentState)); ((LogicalBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_booleanExpression); - setState(211); + setState(207); if (!(precpred(_ctx, 4))) throw new FailedPredicateException(this, "precpred(_ctx, 4)"); - setState(212); + setState(208); ((LogicalBinaryContext)_localctx).operator = match(OR); - setState(213); + setState(209); ((LogicalBinaryContext)_localctx).right = booleanExpression(5); } break; } } } - setState(218); + setState(214); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,8,_ctx); } @@ -1112,48 +1111,48 @@ public final RegexBooleanExpressionContext regexBooleanExpression() throws Recog enterRule(_localctx, 12, RULE_regexBooleanExpression); int _la; try { - setState(233); + setState(229); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,11,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(219); + setState(215); valueExpression(); - setState(221); + setState(217); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(220); + setState(216); match(NOT); } } - setState(223); + setState(219); ((RegexBooleanExpressionContext)_localctx).kind = match(LIKE); - setState(224); + setState(220); ((RegexBooleanExpressionContext)_localctx).pattern = string(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(226); + setState(222); valueExpression(); - setState(228); + setState(224); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(227); + setState(223); match(NOT); } } - setState(230); + setState(226); ((RegexBooleanExpressionContext)_localctx).kind = match(RLIKE); - setState(231); + setState(227); ((RegexBooleanExpressionContext)_localctx).pattern = string(); } break; @@ -1213,23 +1212,23 @@ public final MatchBooleanExpressionContext matchBooleanExpression() throws Recog try { enterOuterAlt(_localctx, 1); { - setState(235); + setState(231); ((MatchBooleanExpressionContext)_localctx).fieldExp = qualifiedName(); - setState(238); + setState(234); _errHandler.sync(this); _la = _input.LA(1); if (_la==CAST_OP) { { - setState(236); + setState(232); match(CAST_OP); - setState(237); + setState(233); ((MatchBooleanExpressionContext)_localctx).fieldType = dataType(); } } - setState(240); + setState(236); match(COLON); - setState(241); + setState(237); ((MatchBooleanExpressionContext)_localctx).matchQuery = constant(); } } @@ -1313,14 +1312,14 @@ public final ValueExpressionContext valueExpression() throws RecognitionExceptio ValueExpressionContext _localctx = new ValueExpressionContext(_ctx, getState()); enterRule(_localctx, 16, RULE_valueExpression); try { - setState(248); + setState(244); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,13,_ctx) ) { case 1: _localctx = new ValueExpressionDefaultContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(243); + setState(239); operatorExpression(0); } break; @@ -1328,11 +1327,11 @@ public final ValueExpressionContext valueExpression() throws RecognitionExceptio _localctx = new ComparisonContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(244); + setState(240); ((ComparisonContext)_localctx).left = operatorExpression(0); - setState(245); + setState(241); comparisonOperator(); - setState(246); + setState(242); ((ComparisonContext)_localctx).right = operatorExpression(0); } break; @@ -1457,7 +1456,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE int _alt; enterOuterAlt(_localctx, 1); { - setState(254); + setState(250); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,14,_ctx) ) { case 1: @@ -1466,7 +1465,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _ctx = _localctx; _prevctx = _localctx; - setState(251); + setState(247); primaryExpression(0); } break; @@ -1475,7 +1474,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _localctx = new ArithmeticUnaryContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(252); + setState(248); ((ArithmeticUnaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { @@ -1486,13 +1485,13 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _errHandler.reportMatch(this); consume(); } - setState(253); + setState(249); operatorExpression(3); } break; } _ctx.stop = _input.LT(-1); - setState(264); + setState(260); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,16,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -1500,7 +1499,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE if ( _parseListeners!=null ) triggerExitRuleEvent(); _prevctx = _localctx; { - setState(262); + setState(258); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,15,_ctx) ) { case 1: @@ -1508,9 +1507,9 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _localctx = new ArithmeticBinaryContext(new OperatorExpressionContext(_parentctx, _parentState)); ((ArithmeticBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_operatorExpression); - setState(256); + setState(252); if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)"); - setState(257); + setState(253); ((ArithmeticBinaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); if ( !(((((_la - 66)) & ~0x3f) == 0 && ((1L << (_la - 66)) & 7L) != 0)) ) { @@ -1521,7 +1520,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _errHandler.reportMatch(this); consume(); } - setState(258); + setState(254); ((ArithmeticBinaryContext)_localctx).right = operatorExpression(3); } break; @@ -1530,9 +1529,9 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _localctx = new ArithmeticBinaryContext(new OperatorExpressionContext(_parentctx, _parentState)); ((ArithmeticBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_operatorExpression); - setState(259); + setState(255); if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)"); - setState(260); + setState(256); ((ArithmeticBinaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { @@ -1543,14 +1542,14 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _errHandler.reportMatch(this); consume(); } - setState(261); + setState(257); ((ArithmeticBinaryContext)_localctx).right = operatorExpression(2); } break; } } } - setState(266); + setState(262); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,16,_ctx); } @@ -1708,7 +1707,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc int _alt; enterOuterAlt(_localctx, 1); { - setState(275); + setState(271); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,17,_ctx) ) { case 1: @@ -1717,7 +1716,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc _ctx = _localctx; _prevctx = _localctx; - setState(268); + setState(264); constant(); } break; @@ -1726,7 +1725,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc _localctx = new DereferenceContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(269); + setState(265); qualifiedName(); } break; @@ -1735,7 +1734,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc _localctx = new FunctionContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(270); + setState(266); functionExpression(); } break; @@ -1744,17 +1743,17 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc _localctx = new ParenthesizedExpressionContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(271); + setState(267); match(LP); - setState(272); + setState(268); booleanExpression(0); - setState(273); + setState(269); match(RP); } break; } _ctx.stop = _input.LT(-1); - setState(282); + setState(278); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,18,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -1765,16 +1764,16 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc { _localctx = new InlineCastContext(new PrimaryExpressionContext(_parentctx, _parentState)); pushNewRecursionContext(_localctx, _startState, RULE_primaryExpression); - setState(277); + setState(273); if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)"); - setState(278); + setState(274); match(CAST_OP); - setState(279); + setState(275); dataType(); } } } - setState(284); + setState(280); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,18,_ctx); } @@ -1836,37 +1835,37 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx try { enterOuterAlt(_localctx, 1); { - setState(285); + setState(281); functionName(); - setState(286); + setState(282); match(LP); - setState(296); + setState(292); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,20,_ctx) ) { case 1: { - setState(287); + setState(283); match(ASTERISK); } break; case 2: { { - setState(288); + setState(284); booleanExpression(0); - setState(293); + setState(289); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(289); + setState(285); match(COMMA); - setState(290); + setState(286); booleanExpression(0); } } - setState(295); + setState(291); _errHandler.sync(this); _la = _input.LA(1); } @@ -1874,7 +1873,7 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx } break; } - setState(298); + setState(294); match(RP); } } @@ -1920,7 +1919,7 @@ public final FunctionNameContext functionName() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(300); + setState(296); identifierOrParameter(); } } @@ -1978,7 +1977,7 @@ public final DataTypeContext dataType() throws RecognitionException { _localctx = new ToDataTypeContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(302); + setState(298); identifier(); } } @@ -2025,9 +2024,9 @@ public final RowCommandContext rowCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(304); + setState(300); match(ROW); - setState(305); + setState(301); fields(); } } @@ -2081,23 +2080,23 @@ public final FieldsContext fields() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(307); + setState(303); field(); - setState(312); + setState(308); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,21,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(308); + setState(304); match(COMMA); - setState(309); + setState(305); field(); } } } - setState(314); + setState(310); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,21,_ctx); } @@ -2149,19 +2148,19 @@ public final FieldContext field() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(318); + setState(314); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,22,_ctx) ) { case 1: { - setState(315); + setState(311); qualifiedName(); - setState(316); + setState(312); match(ASSIGN); } break; } - setState(320); + setState(316); booleanExpression(0); } } @@ -2219,34 +2218,34 @@ public final FromCommandContext fromCommand() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(322); + setState(318); match(FROM); - setState(323); + setState(319); indexPattern(); - setState(328); + setState(324); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,23,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(324); + setState(320); match(COMMA); - setState(325); + setState(321); indexPattern(); } } } - setState(330); + setState(326); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,23,_ctx); } - setState(332); + setState(328); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,24,_ctx) ) { case 1: { - setState(331); + setState(327); metadata(); } break; @@ -2299,19 +2298,19 @@ public final IndexPatternContext indexPattern() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(337); + setState(333); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,25,_ctx) ) { case 1: { - setState(334); + setState(330); clusterString(); - setState(335); + setState(331); match(COLON); } break; } - setState(339); + setState(335); indexString(); } } @@ -2355,7 +2354,7 @@ public final ClusterStringContext clusterString() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(341); + setState(337); match(UNQUOTED_SOURCE); } } @@ -2401,7 +2400,7 @@ public final IndexStringContext indexString() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(343); + setState(339); _la = _input.LA(1); if ( !(_la==QUOTED_STRING || _la==UNQUOTED_SOURCE) ) { _errHandler.recoverInline(this); @@ -2426,11 +2425,14 @@ public final IndexStringContext indexString() throws RecognitionException { @SuppressWarnings("CheckReturnValue") public static class MetadataContext extends ParserRuleContext { - public MetadataOptionContext metadataOption() { - return getRuleContext(MetadataOptionContext.class,0); + public TerminalNode METADATA() { return getToken(EsqlBaseParser.METADATA, 0); } + public List UNQUOTED_SOURCE() { return getTokens(EsqlBaseParser.UNQUOTED_SOURCE); } + public TerminalNode UNQUOTED_SOURCE(int i) { + return getToken(EsqlBaseParser.UNQUOTED_SOURCE, i); } - public Deprecated_metadataContext deprecated_metadata() { - return getRuleContext(Deprecated_metadataContext.class,0); + public List COMMA() { return getTokens(EsqlBaseParser.COMMA); } + public TerminalNode COMMA(int i) { + return getToken(EsqlBaseParser.COMMA, i); } @SuppressWarnings("this-escape") public MetadataContext(ParserRuleContext parent, int invokingState) { @@ -2455,98 +2457,31 @@ public T accept(ParseTreeVisitor visitor) { public final MetadataContext metadata() throws RecognitionException { MetadataContext _localctx = new MetadataContext(_ctx, getState()); enterRule(_localctx, 42, RULE_metadata); - try { - setState(347); - _errHandler.sync(this); - switch (_input.LA(1)) { - case METADATA: - enterOuterAlt(_localctx, 1); - { - setState(345); - metadataOption(); - } - break; - case OPENING_BRACKET: - enterOuterAlt(_localctx, 2); - { - setState(346); - deprecated_metadata(); - } - break; - default: - throw new NoViableAltException(this); - } - } - catch (RecognitionException re) { - _localctx.exception = re; - _errHandler.reportError(this, re); - _errHandler.recover(this, re); - } - finally { - exitRule(); - } - return _localctx; - } - - @SuppressWarnings("CheckReturnValue") - public static class MetadataOptionContext extends ParserRuleContext { - public TerminalNode METADATA() { return getToken(EsqlBaseParser.METADATA, 0); } - public List UNQUOTED_SOURCE() { return getTokens(EsqlBaseParser.UNQUOTED_SOURCE); } - public TerminalNode UNQUOTED_SOURCE(int i) { - return getToken(EsqlBaseParser.UNQUOTED_SOURCE, i); - } - public List COMMA() { return getTokens(EsqlBaseParser.COMMA); } - public TerminalNode COMMA(int i) { - return getToken(EsqlBaseParser.COMMA, i); - } - @SuppressWarnings("this-escape") - public MetadataOptionContext(ParserRuleContext parent, int invokingState) { - super(parent, invokingState); - } - @Override public int getRuleIndex() { return RULE_metadataOption; } - @Override - public void enterRule(ParseTreeListener listener) { - if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).enterMetadataOption(this); - } - @Override - public void exitRule(ParseTreeListener listener) { - if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).exitMetadataOption(this); - } - @Override - public T accept(ParseTreeVisitor visitor) { - if ( visitor instanceof EsqlBaseParserVisitor ) return ((EsqlBaseParserVisitor)visitor).visitMetadataOption(this); - else return visitor.visitChildren(this); - } - } - - public final MetadataOptionContext metadataOption() throws RecognitionException { - MetadataOptionContext _localctx = new MetadataOptionContext(_ctx, getState()); - enterRule(_localctx, 44, RULE_metadataOption); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(349); + setState(341); match(METADATA); - setState(350); + setState(342); match(UNQUOTED_SOURCE); - setState(355); + setState(347); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,27,_ctx); + _alt = getInterpreter().adaptivePredict(_input,26,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(351); + setState(343); match(COMMA); - setState(352); + setState(344); match(UNQUOTED_SOURCE); } } } - setState(357); + setState(349); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,27,_ctx); + _alt = getInterpreter().adaptivePredict(_input,26,_ctx); } } } @@ -2561,57 +2496,6 @@ public final MetadataOptionContext metadataOption() throws RecognitionException return _localctx; } - @SuppressWarnings("CheckReturnValue") - public static class Deprecated_metadataContext extends ParserRuleContext { - public TerminalNode OPENING_BRACKET() { return getToken(EsqlBaseParser.OPENING_BRACKET, 0); } - public MetadataOptionContext metadataOption() { - return getRuleContext(MetadataOptionContext.class,0); - } - public TerminalNode CLOSING_BRACKET() { return getToken(EsqlBaseParser.CLOSING_BRACKET, 0); } - public Deprecated_metadataContext(ParserRuleContext parent, int invokingState) { - super(parent, invokingState); - } - @Override public int getRuleIndex() { return RULE_deprecated_metadata; } - @Override - public void enterRule(ParseTreeListener listener) { - if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).enterDeprecated_metadata(this); - } - @Override - public void exitRule(ParseTreeListener listener) { - if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).exitDeprecated_metadata(this); - } - @Override - public T accept(ParseTreeVisitor visitor) { - if ( visitor instanceof EsqlBaseParserVisitor ) return ((EsqlBaseParserVisitor)visitor).visitDeprecated_metadata(this); - else return visitor.visitChildren(this); - } - } - - public final Deprecated_metadataContext deprecated_metadata() throws RecognitionException { - Deprecated_metadataContext _localctx = new Deprecated_metadataContext(_ctx, getState()); - enterRule(_localctx, 46, RULE_deprecated_metadata); - try { - enterOuterAlt(_localctx, 1); - { - setState(358); - match(OPENING_BRACKET); - setState(359); - metadataOption(); - setState(360); - match(CLOSING_BRACKET); - } - } - catch (RecognitionException re) { - _localctx.exception = re; - _errHandler.reportError(this, re); - _errHandler.recover(this, re); - } - finally { - exitRule(); - } - return _localctx; - } - @SuppressWarnings("CheckReturnValue") public static class MetricsCommandContext extends ParserRuleContext { public AggFieldsContext aggregates; @@ -2656,51 +2540,51 @@ public T accept(ParseTreeVisitor visitor) { public final MetricsCommandContext metricsCommand() throws RecognitionException { MetricsCommandContext _localctx = new MetricsCommandContext(_ctx, getState()); - enterRule(_localctx, 48, RULE_metricsCommand); + enterRule(_localctx, 44, RULE_metricsCommand); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(362); + setState(350); match(DEV_METRICS); - setState(363); + setState(351); indexPattern(); - setState(368); + setState(356); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,28,_ctx); + _alt = getInterpreter().adaptivePredict(_input,27,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(364); + setState(352); match(COMMA); - setState(365); + setState(353); indexPattern(); } } } - setState(370); + setState(358); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,28,_ctx); + _alt = getInterpreter().adaptivePredict(_input,27,_ctx); } - setState(372); + setState(360); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,28,_ctx) ) { case 1: { - setState(371); + setState(359); ((MetricsCommandContext)_localctx).aggregates = aggFields(); } break; } - setState(376); + setState(364); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) { case 1: { - setState(374); + setState(362); match(BY); - setState(375); + setState(363); ((MetricsCommandContext)_localctx).grouping = fields(); } break; @@ -2746,13 +2630,13 @@ public T accept(ParseTreeVisitor visitor) { public final EvalCommandContext evalCommand() throws RecognitionException { EvalCommandContext _localctx = new EvalCommandContext(_ctx, getState()); - enterRule(_localctx, 50, RULE_evalCommand); + enterRule(_localctx, 46, RULE_evalCommand); try { enterOuterAlt(_localctx, 1); { - setState(378); + setState(366); match(EVAL); - setState(379); + setState(367); fields(); } } @@ -2801,30 +2685,30 @@ public T accept(ParseTreeVisitor visitor) { public final StatsCommandContext statsCommand() throws RecognitionException { StatsCommandContext _localctx = new StatsCommandContext(_ctx, getState()); - enterRule(_localctx, 52, RULE_statsCommand); + enterRule(_localctx, 48, RULE_statsCommand); try { enterOuterAlt(_localctx, 1); { - setState(381); + setState(369); match(STATS); - setState(383); + setState(371); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,31,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) { case 1: { - setState(382); + setState(370); ((StatsCommandContext)_localctx).stats = aggFields(); } break; } - setState(387); + setState(375); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,32,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,31,_ctx) ) { case 1: { - setState(385); + setState(373); match(BY); - setState(386); + setState(374); ((StatsCommandContext)_localctx).grouping = fields(); } break; @@ -2876,30 +2760,30 @@ public T accept(ParseTreeVisitor visitor) { public final AggFieldsContext aggFields() throws RecognitionException { AggFieldsContext _localctx = new AggFieldsContext(_ctx, getState()); - enterRule(_localctx, 54, RULE_aggFields); + enterRule(_localctx, 50, RULE_aggFields); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(389); + setState(377); aggField(); - setState(394); + setState(382); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,33,_ctx); + _alt = getInterpreter().adaptivePredict(_input,32,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(390); + setState(378); match(COMMA); - setState(391); + setState(379); aggField(); } } } - setState(396); + setState(384); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,33,_ctx); + _alt = getInterpreter().adaptivePredict(_input,32,_ctx); } } } @@ -2945,20 +2829,20 @@ public T accept(ParseTreeVisitor visitor) { public final AggFieldContext aggField() throws RecognitionException { AggFieldContext _localctx = new AggFieldContext(_ctx, getState()); - enterRule(_localctx, 56, RULE_aggField); + enterRule(_localctx, 52, RULE_aggField); try { enterOuterAlt(_localctx, 1); { - setState(397); + setState(385); field(); - setState(400); + setState(388); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,34,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,33,_ctx) ) { case 1: { - setState(398); + setState(386); match(WHERE); - setState(399); + setState(387); booleanExpression(0); } break; @@ -3010,30 +2894,30 @@ public T accept(ParseTreeVisitor visitor) { public final QualifiedNameContext qualifiedName() throws RecognitionException { QualifiedNameContext _localctx = new QualifiedNameContext(_ctx, getState()); - enterRule(_localctx, 58, RULE_qualifiedName); + enterRule(_localctx, 54, RULE_qualifiedName); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(402); + setState(390); identifierOrParameter(); - setState(407); + setState(395); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,35,_ctx); + _alt = getInterpreter().adaptivePredict(_input,34,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(403); + setState(391); match(DOT); - setState(404); + setState(392); identifierOrParameter(); } } } - setState(409); + setState(397); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,35,_ctx); + _alt = getInterpreter().adaptivePredict(_input,34,_ctx); } } } @@ -3082,30 +2966,30 @@ public T accept(ParseTreeVisitor visitor) { public final QualifiedNamePatternContext qualifiedNamePattern() throws RecognitionException { QualifiedNamePatternContext _localctx = new QualifiedNamePatternContext(_ctx, getState()); - enterRule(_localctx, 60, RULE_qualifiedNamePattern); + enterRule(_localctx, 56, RULE_qualifiedNamePattern); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(410); + setState(398); identifierPattern(); - setState(415); + setState(403); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,36,_ctx); + _alt = getInterpreter().adaptivePredict(_input,35,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(411); + setState(399); match(DOT); - setState(412); + setState(400); identifierPattern(); } } } - setState(417); + setState(405); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,36,_ctx); + _alt = getInterpreter().adaptivePredict(_input,35,_ctx); } } } @@ -3154,30 +3038,30 @@ public T accept(ParseTreeVisitor visitor) { public final QualifiedNamePatternsContext qualifiedNamePatterns() throws RecognitionException { QualifiedNamePatternsContext _localctx = new QualifiedNamePatternsContext(_ctx, getState()); - enterRule(_localctx, 62, RULE_qualifiedNamePatterns); + enterRule(_localctx, 58, RULE_qualifiedNamePatterns); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(418); + setState(406); qualifiedNamePattern(); - setState(423); + setState(411); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,37,_ctx); + _alt = getInterpreter().adaptivePredict(_input,36,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(419); + setState(407); match(COMMA); - setState(420); + setState(408); qualifiedNamePattern(); } } } - setState(425); + setState(413); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,37,_ctx); + _alt = getInterpreter().adaptivePredict(_input,36,_ctx); } } } @@ -3218,12 +3102,12 @@ public T accept(ParseTreeVisitor visitor) { public final IdentifierContext identifier() throws RecognitionException { IdentifierContext _localctx = new IdentifierContext(_ctx, getState()); - enterRule(_localctx, 64, RULE_identifier); + enterRule(_localctx, 60, RULE_identifier); int _la; try { enterOuterAlt(_localctx, 1); { - setState(426); + setState(414); _la = _input.LA(1); if ( !(_la==UNQUOTED_IDENTIFIER || _la==QUOTED_IDENTIFIER) ) { _errHandler.recoverInline(this); @@ -3274,24 +3158,24 @@ public T accept(ParseTreeVisitor visitor) { public final IdentifierPatternContext identifierPattern() throws RecognitionException { IdentifierPatternContext _localctx = new IdentifierPatternContext(_ctx, getState()); - enterRule(_localctx, 66, RULE_identifierPattern); + enterRule(_localctx, 62, RULE_identifierPattern); try { - setState(431); + setState(419); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,38,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,37,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(428); + setState(416); match(ID_PATTERN); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(429); + setState(417); if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(430); + setState(418); parameter(); } break; @@ -3561,17 +3445,17 @@ public T accept(ParseTreeVisitor visitor) { public final ConstantContext constant() throws RecognitionException { ConstantContext _localctx = new ConstantContext(_ctx, getState()); - enterRule(_localctx, 68, RULE_constant); + enterRule(_localctx, 64, RULE_constant); int _la; try { - setState(475); + setState(463); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,42,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,41,_ctx) ) { case 1: _localctx = new NullLiteralContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(433); + setState(421); match(NULL); } break; @@ -3579,9 +3463,9 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new QualifiedIntegerLiteralContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(434); + setState(422); integerValue(); - setState(435); + setState(423); match(UNQUOTED_IDENTIFIER); } break; @@ -3589,7 +3473,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new DecimalLiteralContext(_localctx); enterOuterAlt(_localctx, 3); { - setState(437); + setState(425); decimalValue(); } break; @@ -3597,7 +3481,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new IntegerLiteralContext(_localctx); enterOuterAlt(_localctx, 4); { - setState(438); + setState(426); integerValue(); } break; @@ -3605,7 +3489,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new BooleanLiteralContext(_localctx); enterOuterAlt(_localctx, 5); { - setState(439); + setState(427); booleanValue(); } break; @@ -3613,7 +3497,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new InputParameterContext(_localctx); enterOuterAlt(_localctx, 6); { - setState(440); + setState(428); parameter(); } break; @@ -3621,7 +3505,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new StringLiteralContext(_localctx); enterOuterAlt(_localctx, 7); { - setState(441); + setState(429); string(); } break; @@ -3629,27 +3513,27 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new NumericArrayLiteralContext(_localctx); enterOuterAlt(_localctx, 8); { - setState(442); + setState(430); match(OPENING_BRACKET); - setState(443); + setState(431); numericValue(); - setState(448); + setState(436); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(444); + setState(432); match(COMMA); - setState(445); + setState(433); numericValue(); } } - setState(450); + setState(438); _errHandler.sync(this); _la = _input.LA(1); } - setState(451); + setState(439); match(CLOSING_BRACKET); } break; @@ -3657,27 +3541,27 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new BooleanArrayLiteralContext(_localctx); enterOuterAlt(_localctx, 9); { - setState(453); + setState(441); match(OPENING_BRACKET); - setState(454); + setState(442); booleanValue(); - setState(459); + setState(447); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(455); + setState(443); match(COMMA); - setState(456); + setState(444); booleanValue(); } } - setState(461); + setState(449); _errHandler.sync(this); _la = _input.LA(1); } - setState(462); + setState(450); match(CLOSING_BRACKET); } break; @@ -3685,27 +3569,27 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new StringArrayLiteralContext(_localctx); enterOuterAlt(_localctx, 10); { - setState(464); + setState(452); match(OPENING_BRACKET); - setState(465); + setState(453); string(); - setState(470); + setState(458); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(466); + setState(454); match(COMMA); - setState(467); + setState(455); string(); } } - setState(472); + setState(460); _errHandler.sync(this); _la = _input.LA(1); } - setState(473); + setState(461); match(CLOSING_BRACKET); } break; @@ -3777,16 +3661,16 @@ public T accept(ParseTreeVisitor visitor) { public final ParameterContext parameter() throws RecognitionException { ParameterContext _localctx = new ParameterContext(_ctx, getState()); - enterRule(_localctx, 70, RULE_parameter); + enterRule(_localctx, 66, RULE_parameter); try { - setState(479); + setState(467); _errHandler.sync(this); switch (_input.LA(1)) { case PARAM: _localctx = new InputParamContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(477); + setState(465); match(PARAM); } break; @@ -3794,7 +3678,7 @@ public final ParameterContext parameter() throws RecognitionException { _localctx = new InputNamedOrPositionalParamContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(478); + setState(466); match(NAMED_OR_POSITIONAL_PARAM); } break; @@ -3843,24 +3727,24 @@ public T accept(ParseTreeVisitor visitor) { public final IdentifierOrParameterContext identifierOrParameter() throws RecognitionException { IdentifierOrParameterContext _localctx = new IdentifierOrParameterContext(_ctx, getState()); - enterRule(_localctx, 72, RULE_identifierOrParameter); + enterRule(_localctx, 68, RULE_identifierOrParameter); try { - setState(484); + setState(472); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,44,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,43,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(481); + setState(469); identifier(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(482); + setState(470); if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(483); + setState(471); parameter(); } break; @@ -3903,13 +3787,13 @@ public T accept(ParseTreeVisitor visitor) { public final LimitCommandContext limitCommand() throws RecognitionException { LimitCommandContext _localctx = new LimitCommandContext(_ctx, getState()); - enterRule(_localctx, 74, RULE_limitCommand); + enterRule(_localctx, 70, RULE_limitCommand); try { enterOuterAlt(_localctx, 1); { - setState(486); + setState(474); match(LIMIT); - setState(487); + setState(475); match(INTEGER_LITERAL); } } @@ -3959,32 +3843,32 @@ public T accept(ParseTreeVisitor visitor) { public final SortCommandContext sortCommand() throws RecognitionException { SortCommandContext _localctx = new SortCommandContext(_ctx, getState()); - enterRule(_localctx, 76, RULE_sortCommand); + enterRule(_localctx, 72, RULE_sortCommand); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(489); + setState(477); match(SORT); - setState(490); + setState(478); orderExpression(); - setState(495); + setState(483); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,45,_ctx); + _alt = getInterpreter().adaptivePredict(_input,44,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(491); + setState(479); match(COMMA); - setState(492); + setState(480); orderExpression(); } } } - setState(497); + setState(485); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,45,_ctx); + _alt = getInterpreter().adaptivePredict(_input,44,_ctx); } } } @@ -4033,19 +3917,19 @@ public T accept(ParseTreeVisitor visitor) { public final OrderExpressionContext orderExpression() throws RecognitionException { OrderExpressionContext _localctx = new OrderExpressionContext(_ctx, getState()); - enterRule(_localctx, 78, RULE_orderExpression); + enterRule(_localctx, 74, RULE_orderExpression); int _la; try { enterOuterAlt(_localctx, 1); { - setState(498); + setState(486); booleanExpression(0); - setState(500); + setState(488); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,46,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,45,_ctx) ) { case 1: { - setState(499); + setState(487); ((OrderExpressionContext)_localctx).ordering = _input.LT(1); _la = _input.LA(1); if ( !(_la==ASC || _la==DESC) ) { @@ -4059,14 +3943,14 @@ public final OrderExpressionContext orderExpression() throws RecognitionExceptio } break; } - setState(504); + setState(492); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,47,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,46,_ctx) ) { case 1: { - setState(502); + setState(490); match(NULLS); - setState(503); + setState(491); ((OrderExpressionContext)_localctx).nullOrdering = _input.LT(1); _la = _input.LA(1); if ( !(_la==FIRST || _la==LAST) ) { @@ -4121,13 +4005,13 @@ public T accept(ParseTreeVisitor visitor) { public final KeepCommandContext keepCommand() throws RecognitionException { KeepCommandContext _localctx = new KeepCommandContext(_ctx, getState()); - enterRule(_localctx, 80, RULE_keepCommand); + enterRule(_localctx, 76, RULE_keepCommand); try { enterOuterAlt(_localctx, 1); { - setState(506); + setState(494); match(KEEP); - setState(507); + setState(495); qualifiedNamePatterns(); } } @@ -4170,13 +4054,13 @@ public T accept(ParseTreeVisitor visitor) { public final DropCommandContext dropCommand() throws RecognitionException { DropCommandContext _localctx = new DropCommandContext(_ctx, getState()); - enterRule(_localctx, 82, RULE_dropCommand); + enterRule(_localctx, 78, RULE_dropCommand); try { enterOuterAlt(_localctx, 1); { - setState(509); + setState(497); match(DROP); - setState(510); + setState(498); qualifiedNamePatterns(); } } @@ -4226,32 +4110,32 @@ public T accept(ParseTreeVisitor visitor) { public final RenameCommandContext renameCommand() throws RecognitionException { RenameCommandContext _localctx = new RenameCommandContext(_ctx, getState()); - enterRule(_localctx, 84, RULE_renameCommand); + enterRule(_localctx, 80, RULE_renameCommand); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(512); + setState(500); match(RENAME); - setState(513); + setState(501); renameClause(); - setState(518); + setState(506); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,48,_ctx); + _alt = getInterpreter().adaptivePredict(_input,47,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(514); + setState(502); match(COMMA); - setState(515); + setState(503); renameClause(); } } } - setState(520); + setState(508); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,48,_ctx); + _alt = getInterpreter().adaptivePredict(_input,47,_ctx); } } } @@ -4299,15 +4183,15 @@ public T accept(ParseTreeVisitor visitor) { public final RenameClauseContext renameClause() throws RecognitionException { RenameClauseContext _localctx = new RenameClauseContext(_ctx, getState()); - enterRule(_localctx, 86, RULE_renameClause); + enterRule(_localctx, 82, RULE_renameClause); try { enterOuterAlt(_localctx, 1); { - setState(521); + setState(509); ((RenameClauseContext)_localctx).oldName = qualifiedNamePattern(); - setState(522); + setState(510); match(AS); - setState(523); + setState(511); ((RenameClauseContext)_localctx).newName = qualifiedNamePattern(); } } @@ -4356,22 +4240,22 @@ public T accept(ParseTreeVisitor visitor) { public final DissectCommandContext dissectCommand() throws RecognitionException { DissectCommandContext _localctx = new DissectCommandContext(_ctx, getState()); - enterRule(_localctx, 88, RULE_dissectCommand); + enterRule(_localctx, 84, RULE_dissectCommand); try { enterOuterAlt(_localctx, 1); { - setState(525); + setState(513); match(DISSECT); - setState(526); + setState(514); primaryExpression(0); - setState(527); + setState(515); string(); - setState(529); + setState(517); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,49,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,48,_ctx) ) { case 1: { - setState(528); + setState(516); commandOptions(); } break; @@ -4420,15 +4304,15 @@ public T accept(ParseTreeVisitor visitor) { public final GrokCommandContext grokCommand() throws RecognitionException { GrokCommandContext _localctx = new GrokCommandContext(_ctx, getState()); - enterRule(_localctx, 90, RULE_grokCommand); + enterRule(_localctx, 86, RULE_grokCommand); try { enterOuterAlt(_localctx, 1); { - setState(531); + setState(519); match(GROK); - setState(532); + setState(520); primaryExpression(0); - setState(533); + setState(521); string(); } } @@ -4471,13 +4355,13 @@ public T accept(ParseTreeVisitor visitor) { public final MvExpandCommandContext mvExpandCommand() throws RecognitionException { MvExpandCommandContext _localctx = new MvExpandCommandContext(_ctx, getState()); - enterRule(_localctx, 92, RULE_mvExpandCommand); + enterRule(_localctx, 88, RULE_mvExpandCommand); try { enterOuterAlt(_localctx, 1); { - setState(535); + setState(523); match(MV_EXPAND); - setState(536); + setState(524); qualifiedName(); } } @@ -4526,30 +4410,30 @@ public T accept(ParseTreeVisitor visitor) { public final CommandOptionsContext commandOptions() throws RecognitionException { CommandOptionsContext _localctx = new CommandOptionsContext(_ctx, getState()); - enterRule(_localctx, 94, RULE_commandOptions); + enterRule(_localctx, 90, RULE_commandOptions); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(538); + setState(526); commandOption(); - setState(543); + setState(531); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,50,_ctx); + _alt = getInterpreter().adaptivePredict(_input,49,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(539); + setState(527); match(COMMA); - setState(540); + setState(528); commandOption(); } } } - setState(545); + setState(533); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,50,_ctx); + _alt = getInterpreter().adaptivePredict(_input,49,_ctx); } } } @@ -4595,15 +4479,15 @@ public T accept(ParseTreeVisitor visitor) { public final CommandOptionContext commandOption() throws RecognitionException { CommandOptionContext _localctx = new CommandOptionContext(_ctx, getState()); - enterRule(_localctx, 96, RULE_commandOption); + enterRule(_localctx, 92, RULE_commandOption); try { enterOuterAlt(_localctx, 1); { - setState(546); + setState(534); identifier(); - setState(547); + setState(535); match(ASSIGN); - setState(548); + setState(536); constant(); } } @@ -4644,12 +4528,12 @@ public T accept(ParseTreeVisitor visitor) { public final BooleanValueContext booleanValue() throws RecognitionException { BooleanValueContext _localctx = new BooleanValueContext(_ctx, getState()); - enterRule(_localctx, 98, RULE_booleanValue); + enterRule(_localctx, 94, RULE_booleanValue); int _la; try { enterOuterAlt(_localctx, 1); { - setState(550); + setState(538); _la = _input.LA(1); if ( !(_la==FALSE || _la==TRUE) ) { _errHandler.recoverInline(this); @@ -4702,22 +4586,22 @@ public T accept(ParseTreeVisitor visitor) { public final NumericValueContext numericValue() throws RecognitionException { NumericValueContext _localctx = new NumericValueContext(_ctx, getState()); - enterRule(_localctx, 100, RULE_numericValue); + enterRule(_localctx, 96, RULE_numericValue); try { - setState(554); + setState(542); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,51,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,50,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(552); + setState(540); decimalValue(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(553); + setState(541); integerValue(); } break; @@ -4761,17 +4645,17 @@ public T accept(ParseTreeVisitor visitor) { public final DecimalValueContext decimalValue() throws RecognitionException { DecimalValueContext _localctx = new DecimalValueContext(_ctx, getState()); - enterRule(_localctx, 102, RULE_decimalValue); + enterRule(_localctx, 98, RULE_decimalValue); int _la; try { enterOuterAlt(_localctx, 1); { - setState(557); + setState(545); _errHandler.sync(this); _la = _input.LA(1); if (_la==PLUS || _la==MINUS) { { - setState(556); + setState(544); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { _errHandler.recoverInline(this); @@ -4784,7 +4668,7 @@ public final DecimalValueContext decimalValue() throws RecognitionException { } } - setState(559); + setState(547); match(DECIMAL_LITERAL); } } @@ -4826,17 +4710,17 @@ public T accept(ParseTreeVisitor visitor) { public final IntegerValueContext integerValue() throws RecognitionException { IntegerValueContext _localctx = new IntegerValueContext(_ctx, getState()); - enterRule(_localctx, 104, RULE_integerValue); + enterRule(_localctx, 100, RULE_integerValue); int _la; try { enterOuterAlt(_localctx, 1); { - setState(562); + setState(550); _errHandler.sync(this); _la = _input.LA(1); if (_la==PLUS || _la==MINUS) { { - setState(561); + setState(549); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { _errHandler.recoverInline(this); @@ -4849,7 +4733,7 @@ public final IntegerValueContext integerValue() throws RecognitionException { } } - setState(564); + setState(552); match(INTEGER_LITERAL); } } @@ -4889,11 +4773,11 @@ public T accept(ParseTreeVisitor visitor) { public final StringContext string() throws RecognitionException { StringContext _localctx = new StringContext(_ctx, getState()); - enterRule(_localctx, 106, RULE_string); + enterRule(_localctx, 102, RULE_string); try { enterOuterAlt(_localctx, 1); { - setState(566); + setState(554); match(QUOTED_STRING); } } @@ -4938,12 +4822,12 @@ public T accept(ParseTreeVisitor visitor) { public final ComparisonOperatorContext comparisonOperator() throws RecognitionException { ComparisonOperatorContext _localctx = new ComparisonOperatorContext(_ctx, getState()); - enterRule(_localctx, 108, RULE_comparisonOperator); + enterRule(_localctx, 104, RULE_comparisonOperator); int _la; try { enterOuterAlt(_localctx, 1); { - setState(568); + setState(556); _la = _input.LA(1); if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & -432345564227567616L) != 0)) ) { _errHandler.recoverInline(this); @@ -4994,13 +4878,13 @@ public T accept(ParseTreeVisitor visitor) { public final ExplainCommandContext explainCommand() throws RecognitionException { ExplainCommandContext _localctx = new ExplainCommandContext(_ctx, getState()); - enterRule(_localctx, 110, RULE_explainCommand); + enterRule(_localctx, 106, RULE_explainCommand); try { enterOuterAlt(_localctx, 1); { - setState(570); + setState(558); match(EXPLAIN); - setState(571); + setState(559); subqueryExpression(); } } @@ -5044,15 +4928,15 @@ public T accept(ParseTreeVisitor visitor) { public final SubqueryExpressionContext subqueryExpression() throws RecognitionException { SubqueryExpressionContext _localctx = new SubqueryExpressionContext(_ctx, getState()); - enterRule(_localctx, 112, RULE_subqueryExpression); + enterRule(_localctx, 108, RULE_subqueryExpression); try { enterOuterAlt(_localctx, 1); { - setState(573); + setState(561); match(OPENING_BRACKET); - setState(574); + setState(562); query(0); - setState(575); + setState(563); match(CLOSING_BRACKET); } } @@ -5104,14 +4988,14 @@ public T accept(ParseTreeVisitor visitor) { public final ShowCommandContext showCommand() throws RecognitionException { ShowCommandContext _localctx = new ShowCommandContext(_ctx, getState()); - enterRule(_localctx, 114, RULE_showCommand); + enterRule(_localctx, 110, RULE_showCommand); try { _localctx = new ShowInfoContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(577); + setState(565); match(SHOW); - setState(578); + setState(566); match(INFO); } } @@ -5169,53 +5053,53 @@ public T accept(ParseTreeVisitor visitor) { public final EnrichCommandContext enrichCommand() throws RecognitionException { EnrichCommandContext _localctx = new EnrichCommandContext(_ctx, getState()); - enterRule(_localctx, 116, RULE_enrichCommand); + enterRule(_localctx, 112, RULE_enrichCommand); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(580); + setState(568); match(ENRICH); - setState(581); + setState(569); ((EnrichCommandContext)_localctx).policyName = match(ENRICH_POLICY_NAME); - setState(584); + setState(572); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,54,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,53,_ctx) ) { case 1: { - setState(582); + setState(570); match(ON); - setState(583); + setState(571); ((EnrichCommandContext)_localctx).matchField = qualifiedNamePattern(); } break; } - setState(595); + setState(583); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,56,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,55,_ctx) ) { case 1: { - setState(586); + setState(574); match(WITH); - setState(587); + setState(575); enrichWithClause(); - setState(592); + setState(580); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,55,_ctx); + _alt = getInterpreter().adaptivePredict(_input,54,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(588); + setState(576); match(COMMA); - setState(589); + setState(577); enrichWithClause(); } } } - setState(594); + setState(582); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,55,_ctx); + _alt = getInterpreter().adaptivePredict(_input,54,_ctx); } } break; @@ -5266,23 +5150,23 @@ public T accept(ParseTreeVisitor visitor) { public final EnrichWithClauseContext enrichWithClause() throws RecognitionException { EnrichWithClauseContext _localctx = new EnrichWithClauseContext(_ctx, getState()); - enterRule(_localctx, 118, RULE_enrichWithClause); + enterRule(_localctx, 114, RULE_enrichWithClause); try { enterOuterAlt(_localctx, 1); { - setState(600); + setState(588); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,57,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,56,_ctx) ) { case 1: { - setState(597); + setState(585); ((EnrichWithClauseContext)_localctx).newName = qualifiedNamePattern(); - setState(598); + setState(586); match(ASSIGN); } break; } - setState(602); + setState(590); ((EnrichWithClauseContext)_localctx).enrichField = qualifiedNamePattern(); } } @@ -5331,17 +5215,17 @@ public T accept(ParseTreeVisitor visitor) { public final LookupCommandContext lookupCommand() throws RecognitionException { LookupCommandContext _localctx = new LookupCommandContext(_ctx, getState()); - enterRule(_localctx, 120, RULE_lookupCommand); + enterRule(_localctx, 116, RULE_lookupCommand); try { enterOuterAlt(_localctx, 1); { - setState(604); + setState(592); match(DEV_LOOKUP); - setState(605); + setState(593); ((LookupCommandContext)_localctx).tableName = indexPattern(); - setState(606); + setState(594); match(ON); - setState(607); + setState(595); ((LookupCommandContext)_localctx).matchFields = qualifiedNamePatterns(); } } @@ -5390,22 +5274,22 @@ public T accept(ParseTreeVisitor visitor) { public final InlinestatsCommandContext inlinestatsCommand() throws RecognitionException { InlinestatsCommandContext _localctx = new InlinestatsCommandContext(_ctx, getState()); - enterRule(_localctx, 122, RULE_inlinestatsCommand); + enterRule(_localctx, 118, RULE_inlinestatsCommand); try { enterOuterAlt(_localctx, 1); { - setState(609); + setState(597); match(DEV_INLINESTATS); - setState(610); + setState(598); ((InlinestatsCommandContext)_localctx).stats = aggFields(); - setState(613); + setState(601); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,58,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,57,_ctx) ) { case 1: { - setState(611); + setState(599); match(BY); - setState(612); + setState(600); ((InlinestatsCommandContext)_localctx).grouping = fields(); } break; @@ -5458,17 +5342,17 @@ public T accept(ParseTreeVisitor visitor) { public final JoinCommandContext joinCommand() throws RecognitionException { JoinCommandContext _localctx = new JoinCommandContext(_ctx, getState()); - enterRule(_localctx, 124, RULE_joinCommand); + enterRule(_localctx, 120, RULE_joinCommand); int _la; try { enterOuterAlt(_localctx, 1); { - setState(616); + setState(604); _errHandler.sync(this); _la = _input.LA(1); if ((((_la) & ~0x3f) == 0 && ((1L << _la) & 29360128L) != 0)) { { - setState(615); + setState(603); ((JoinCommandContext)_localctx).type = _input.LT(1); _la = _input.LA(1); if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & 29360128L) != 0)) ) { @@ -5482,11 +5366,11 @@ public final JoinCommandContext joinCommand() throws RecognitionException { } } - setState(618); + setState(606); match(DEV_JOIN); - setState(619); + setState(607); joinTarget(); - setState(620); + setState(608); joinCondition(); } } @@ -5534,21 +5418,21 @@ public T accept(ParseTreeVisitor visitor) { public final JoinTargetContext joinTarget() throws RecognitionException { JoinTargetContext _localctx = new JoinTargetContext(_ctx, getState()); - enterRule(_localctx, 126, RULE_joinTarget); + enterRule(_localctx, 122, RULE_joinTarget); int _la; try { enterOuterAlt(_localctx, 1); { - setState(622); + setState(610); ((JoinTargetContext)_localctx).index = identifier(); - setState(625); + setState(613); _errHandler.sync(this); _la = _input.LA(1); if (_la==AS) { { - setState(623); + setState(611); match(AS); - setState(624); + setState(612); ((JoinTargetContext)_localctx).alias = identifier(); } } @@ -5601,32 +5485,32 @@ public T accept(ParseTreeVisitor visitor) { public final JoinConditionContext joinCondition() throws RecognitionException { JoinConditionContext _localctx = new JoinConditionContext(_ctx, getState()); - enterRule(_localctx, 128, RULE_joinCondition); + enterRule(_localctx, 124, RULE_joinCondition); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(627); + setState(615); match(ON); - setState(628); + setState(616); joinPredicate(); - setState(633); + setState(621); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,61,_ctx); + _alt = getInterpreter().adaptivePredict(_input,60,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(629); + setState(617); match(COMMA); - setState(630); + setState(618); joinPredicate(); } } } - setState(635); + setState(623); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,61,_ctx); + _alt = getInterpreter().adaptivePredict(_input,60,_ctx); } } } @@ -5668,11 +5552,11 @@ public T accept(ParseTreeVisitor visitor) { public final JoinPredicateContext joinPredicate() throws RecognitionException { JoinPredicateContext _localctx = new JoinPredicateContext(_ctx, getState()); - enterRule(_localctx, 130, RULE_joinPredicate); + enterRule(_localctx, 126, RULE_joinPredicate); try { enterOuterAlt(_localctx, 1); { - setState(636); + setState(624); valueExpression(); } } @@ -5701,9 +5585,9 @@ public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) { return operatorExpression_sempred((OperatorExpressionContext)_localctx, predIndex); case 10: return primaryExpression_sempred((PrimaryExpressionContext)_localctx, predIndex); - case 33: + case 31: return identifierPattern_sempred((IdentifierPatternContext)_localctx, predIndex); - case 36: + case 34: return identifierOrParameter_sempred((IdentifierOrParameterContext)_localctx, predIndex); } return true; @@ -5774,7 +5658,7 @@ private boolean identifierOrParameter_sempred(IdentifierOrParameterContext _loca } public static final String _serializedATN = - "\u0004\u0001\u0080\u027f\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001"+ + "\u0004\u0001\u0080\u0273\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001"+ "\u0002\u0002\u0007\u0002\u0002\u0003\u0007\u0003\u0002\u0004\u0007\u0004"+ "\u0002\u0005\u0007\u0005\u0002\u0006\u0007\u0006\u0002\u0007\u0007\u0007"+ "\u0002\b\u0007\b\u0002\t\u0007\t\u0002\n\u0007\n\u0002\u000b\u0007\u000b"+ @@ -5790,392 +5674,385 @@ private boolean identifierOrParameter_sempred(IdentifierOrParameterContext _loca ",\u0002-\u0007-\u0002.\u0007.\u0002/\u0007/\u00020\u00070\u00021\u0007"+ "1\u00022\u00072\u00023\u00073\u00024\u00074\u00025\u00075\u00026\u0007"+ "6\u00027\u00077\u00028\u00078\u00029\u00079\u0002:\u0007:\u0002;\u0007"+ - ";\u0002<\u0007<\u0002=\u0007=\u0002>\u0007>\u0002?\u0007?\u0002@\u0007"+ - "@\u0002A\u0007A\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0001\u0001"+ - "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0005\u0001\u008e"+ - "\b\u0001\n\u0001\f\u0001\u0091\t\u0001\u0001\u0002\u0001\u0002\u0001\u0002"+ - "\u0001\u0002\u0001\u0002\u0001\u0002\u0003\u0002\u0099\b\u0002\u0001\u0003"+ - "\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003"+ - "\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003"+ - "\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0003\u0003"+ - "\u00ad\b\u0003\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0005\u0001\u0005"+ - "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003\u0005"+ - "\u00b9\b\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005"+ - "\u0005\u0005\u00c0\b\u0005\n\u0005\f\u0005\u00c3\t\u0005\u0001\u0005\u0001"+ - "\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00ca\b\u0005\u0001"+ - "\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00cf\b\u0005\u0001\u0005\u0001"+ - "\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0005\u0005\u00d7"+ - "\b\u0005\n\u0005\f\u0005\u00da\t\u0005\u0001\u0006\u0001\u0006\u0003\u0006"+ - "\u00de\b\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006"+ - "\u0003\u0006\u00e5\b\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0003\u0006"+ - "\u00ea\b\u0006\u0001\u0007\u0001\u0007\u0001\u0007\u0003\u0007\u00ef\b"+ - "\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\b\u0001\b\u0001\b\u0001"+ - "\b\u0001\b\u0003\b\u00f9\b\b\u0001\t\u0001\t\u0001\t\u0001\t\u0003\t\u00ff"+ - "\b\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0005\t\u0107\b\t"+ - "\n\t\f\t\u010a\t\t\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001"+ - "\n\u0001\n\u0003\n\u0114\b\n\u0001\n\u0001\n\u0001\n\u0005\n\u0119\b\n"+ - "\n\n\f\n\u011c\t\n\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001"+ - "\u000b\u0001\u000b\u0005\u000b\u0124\b\u000b\n\u000b\f\u000b\u0127\t\u000b"+ - "\u0003\u000b\u0129\b\u000b\u0001\u000b\u0001\u000b\u0001\f\u0001\f\u0001"+ - "\r\u0001\r\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000f\u0001\u000f"+ - "\u0001\u000f\u0005\u000f\u0137\b\u000f\n\u000f\f\u000f\u013a\t\u000f\u0001"+ - "\u0010\u0001\u0010\u0001\u0010\u0003\u0010\u013f\b\u0010\u0001\u0010\u0001"+ - "\u0010\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0005\u0011\u0147"+ - "\b\u0011\n\u0011\f\u0011\u014a\t\u0011\u0001\u0011\u0003\u0011\u014d\b"+ - "\u0011\u0001\u0012\u0001\u0012\u0001\u0012\u0003\u0012\u0152\b\u0012\u0001"+ - "\u0012\u0001\u0012\u0001\u0013\u0001\u0013\u0001\u0014\u0001\u0014\u0001"+ - "\u0015\u0001\u0015\u0003\u0015\u015c\b\u0015\u0001\u0016\u0001\u0016\u0001"+ - "\u0016\u0001\u0016\u0005\u0016\u0162\b\u0016\n\u0016\f\u0016\u0165\t\u0016"+ - "\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0018\u0001\u0018"+ - "\u0001\u0018\u0001\u0018\u0005\u0018\u016f\b\u0018\n\u0018\f\u0018\u0172"+ - "\t\u0018\u0001\u0018\u0003\u0018\u0175\b\u0018\u0001\u0018\u0001\u0018"+ - "\u0003\u0018\u0179\b\u0018\u0001\u0019\u0001\u0019\u0001\u0019\u0001\u001a"+ - "\u0001\u001a\u0003\u001a\u0180\b\u001a\u0001\u001a\u0001\u001a\u0003\u001a"+ - "\u0184\b\u001a\u0001\u001b\u0001\u001b\u0001\u001b\u0005\u001b\u0189\b"+ - "\u001b\n\u001b\f\u001b\u018c\t\u001b\u0001\u001c\u0001\u001c\u0001\u001c"+ - "\u0003\u001c\u0191\b\u001c\u0001\u001d\u0001\u001d\u0001\u001d\u0005\u001d"+ - "\u0196\b\u001d\n\u001d\f\u001d\u0199\t\u001d\u0001\u001e\u0001\u001e\u0001"+ - "\u001e\u0005\u001e\u019e\b\u001e\n\u001e\f\u001e\u01a1\t\u001e\u0001\u001f"+ - "\u0001\u001f\u0001\u001f\u0005\u001f\u01a6\b\u001f\n\u001f\f\u001f\u01a9"+ - "\t\u001f\u0001 \u0001 \u0001!\u0001!\u0001!\u0003!\u01b0\b!\u0001\"\u0001"+ - "\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001"+ - "\"\u0001\"\u0001\"\u0005\"\u01bf\b\"\n\"\f\"\u01c2\t\"\u0001\"\u0001\""+ - "\u0001\"\u0001\"\u0001\"\u0001\"\u0005\"\u01ca\b\"\n\"\f\"\u01cd\t\"\u0001"+ - "\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0005\"\u01d5\b\"\n\"\f\"\u01d8"+ - "\t\"\u0001\"\u0001\"\u0003\"\u01dc\b\"\u0001#\u0001#\u0003#\u01e0\b#\u0001"+ - "$\u0001$\u0001$\u0003$\u01e5\b$\u0001%\u0001%\u0001%\u0001&\u0001&\u0001"+ - "&\u0001&\u0005&\u01ee\b&\n&\f&\u01f1\t&\u0001\'\u0001\'\u0003\'\u01f5"+ - "\b\'\u0001\'\u0001\'\u0003\'\u01f9\b\'\u0001(\u0001(\u0001(\u0001)\u0001"+ - ")\u0001)\u0001*\u0001*\u0001*\u0001*\u0005*\u0205\b*\n*\f*\u0208\t*\u0001"+ - "+\u0001+\u0001+\u0001+\u0001,\u0001,\u0001,\u0001,\u0003,\u0212\b,\u0001"+ - "-\u0001-\u0001-\u0001-\u0001.\u0001.\u0001.\u0001/\u0001/\u0001/\u0005"+ - "/\u021e\b/\n/\f/\u0221\t/\u00010\u00010\u00010\u00010\u00011\u00011\u0001"+ - "2\u00012\u00032\u022b\b2\u00013\u00033\u022e\b3\u00013\u00013\u00014\u0003"+ - "4\u0233\b4\u00014\u00014\u00015\u00015\u00016\u00016\u00017\u00017\u0001"+ - "7\u00018\u00018\u00018\u00018\u00019\u00019\u00019\u0001:\u0001:\u0001"+ - ":\u0001:\u0003:\u0249\b:\u0001:\u0001:\u0001:\u0001:\u0005:\u024f\b:\n"+ - ":\f:\u0252\t:\u0003:\u0254\b:\u0001;\u0001;\u0001;\u0003;\u0259\b;\u0001"+ - ";\u0001;\u0001<\u0001<\u0001<\u0001<\u0001<\u0001=\u0001=\u0001=\u0001"+ - "=\u0003=\u0266\b=\u0001>\u0003>\u0269\b>\u0001>\u0001>\u0001>\u0001>\u0001"+ - "?\u0001?\u0001?\u0003?\u0272\b?\u0001@\u0001@\u0001@\u0001@\u0005@\u0278"+ - "\b@\n@\f@\u027b\t@\u0001A\u0001A\u0001A\u0000\u0004\u0002\n\u0012\u0014"+ - "B\u0000\u0002\u0004\u0006\b\n\f\u000e\u0010\u0012\u0014\u0016\u0018\u001a"+ - "\u001c\u001e \"$&(*,.02468:<>@BDFHJLNPRTVXZ\\^`bdfhjlnprtvxz|~\u0080\u0082"+ - "\u0000\t\u0001\u0000@A\u0001\u0000BD\u0002\u0000\u001e\u001eQQ\u0001\u0000"+ - "HI\u0002\u0000##((\u0002\u0000++..\u0002\u0000**88\u0002\u000099;?\u0001"+ - "\u0000\u0016\u0018\u0299\u0000\u0084\u0001\u0000\u0000\u0000\u0002\u0087"+ - "\u0001\u0000\u0000\u0000\u0004\u0098\u0001\u0000\u0000\u0000\u0006\u00ac"+ - "\u0001\u0000\u0000\u0000\b\u00ae\u0001\u0000\u0000\u0000\n\u00ce\u0001"+ - "\u0000\u0000\u0000\f\u00e9\u0001\u0000\u0000\u0000\u000e\u00eb\u0001\u0000"+ - "\u0000\u0000\u0010\u00f8\u0001\u0000\u0000\u0000\u0012\u00fe\u0001\u0000"+ - "\u0000\u0000\u0014\u0113\u0001\u0000\u0000\u0000\u0016\u011d\u0001\u0000"+ - "\u0000\u0000\u0018\u012c\u0001\u0000\u0000\u0000\u001a\u012e\u0001\u0000"+ - "\u0000\u0000\u001c\u0130\u0001\u0000\u0000\u0000\u001e\u0133\u0001\u0000"+ - "\u0000\u0000 \u013e\u0001\u0000\u0000\u0000\"\u0142\u0001\u0000\u0000"+ - "\u0000$\u0151\u0001\u0000\u0000\u0000&\u0155\u0001\u0000\u0000\u0000("+ - "\u0157\u0001\u0000\u0000\u0000*\u015b\u0001\u0000\u0000\u0000,\u015d\u0001"+ - "\u0000\u0000\u0000.\u0166\u0001\u0000\u0000\u00000\u016a\u0001\u0000\u0000"+ - "\u00002\u017a\u0001\u0000\u0000\u00004\u017d\u0001\u0000\u0000\u00006"+ - "\u0185\u0001\u0000\u0000\u00008\u018d\u0001\u0000\u0000\u0000:\u0192\u0001"+ - "\u0000\u0000\u0000<\u019a\u0001\u0000\u0000\u0000>\u01a2\u0001\u0000\u0000"+ - "\u0000@\u01aa\u0001\u0000\u0000\u0000B\u01af\u0001\u0000\u0000\u0000D"+ - "\u01db\u0001\u0000\u0000\u0000F\u01df\u0001\u0000\u0000\u0000H\u01e4\u0001"+ - "\u0000\u0000\u0000J\u01e6\u0001\u0000\u0000\u0000L\u01e9\u0001\u0000\u0000"+ - "\u0000N\u01f2\u0001\u0000\u0000\u0000P\u01fa\u0001\u0000\u0000\u0000R"+ - "\u01fd\u0001\u0000\u0000\u0000T\u0200\u0001\u0000\u0000\u0000V\u0209\u0001"+ - "\u0000\u0000\u0000X\u020d\u0001\u0000\u0000\u0000Z\u0213\u0001\u0000\u0000"+ - "\u0000\\\u0217\u0001\u0000\u0000\u0000^\u021a\u0001\u0000\u0000\u0000"+ - "`\u0222\u0001\u0000\u0000\u0000b\u0226\u0001\u0000\u0000\u0000d\u022a"+ - "\u0001\u0000\u0000\u0000f\u022d\u0001\u0000\u0000\u0000h\u0232\u0001\u0000"+ - "\u0000\u0000j\u0236\u0001\u0000\u0000\u0000l\u0238\u0001\u0000\u0000\u0000"+ - "n\u023a\u0001\u0000\u0000\u0000p\u023d\u0001\u0000\u0000\u0000r\u0241"+ - "\u0001\u0000\u0000\u0000t\u0244\u0001\u0000\u0000\u0000v\u0258\u0001\u0000"+ - "\u0000\u0000x\u025c\u0001\u0000\u0000\u0000z\u0261\u0001\u0000\u0000\u0000"+ - "|\u0268\u0001\u0000\u0000\u0000~\u026e\u0001\u0000\u0000\u0000\u0080\u0273"+ - "\u0001\u0000\u0000\u0000\u0082\u027c\u0001\u0000\u0000\u0000\u0084\u0085"+ - "\u0003\u0002\u0001\u0000\u0085\u0086\u0005\u0000\u0000\u0001\u0086\u0001"+ - "\u0001\u0000\u0000\u0000\u0087\u0088\u0006\u0001\uffff\uffff\u0000\u0088"+ - "\u0089\u0003\u0004\u0002\u0000\u0089\u008f\u0001\u0000\u0000\u0000\u008a"+ - "\u008b\n\u0001\u0000\u0000\u008b\u008c\u0005\u001d\u0000\u0000\u008c\u008e"+ - "\u0003\u0006\u0003\u0000\u008d\u008a\u0001\u0000\u0000\u0000\u008e\u0091"+ - "\u0001\u0000\u0000\u0000\u008f\u008d\u0001\u0000\u0000\u0000\u008f\u0090"+ - "\u0001\u0000\u0000\u0000\u0090\u0003\u0001\u0000\u0000\u0000\u0091\u008f"+ - "\u0001\u0000\u0000\u0000\u0092\u0099\u0003n7\u0000\u0093\u0099\u0003\""+ - "\u0011\u0000\u0094\u0099\u0003\u001c\u000e\u0000\u0095\u0099\u0003r9\u0000"+ - "\u0096\u0097\u0004\u0002\u0001\u0000\u0097\u0099\u00030\u0018\u0000\u0098"+ - "\u0092\u0001\u0000\u0000\u0000\u0098\u0093\u0001\u0000\u0000\u0000\u0098"+ - "\u0094\u0001\u0000\u0000\u0000\u0098\u0095\u0001\u0000\u0000\u0000\u0098"+ - "\u0096\u0001\u0000\u0000\u0000\u0099\u0005\u0001\u0000\u0000\u0000\u009a"+ - "\u00ad\u00032\u0019\u0000\u009b\u00ad\u0003\b\u0004\u0000\u009c\u00ad"+ - "\u0003P(\u0000\u009d\u00ad\u0003J%\u0000\u009e\u00ad\u00034\u001a\u0000"+ - "\u009f\u00ad\u0003L&\u0000\u00a0\u00ad\u0003R)\u0000\u00a1\u00ad\u0003"+ - "T*\u0000\u00a2\u00ad\u0003X,\u0000\u00a3\u00ad\u0003Z-\u0000\u00a4\u00ad"+ - "\u0003t:\u0000\u00a5\u00ad\u0003\\.\u0000\u00a6\u00a7\u0004\u0003\u0002"+ - "\u0000\u00a7\u00ad\u0003z=\u0000\u00a8\u00a9\u0004\u0003\u0003\u0000\u00a9"+ - "\u00ad\u0003x<\u0000\u00aa\u00ab\u0004\u0003\u0004\u0000\u00ab\u00ad\u0003"+ - "|>\u0000\u00ac\u009a\u0001\u0000\u0000\u0000\u00ac\u009b\u0001\u0000\u0000"+ - "\u0000\u00ac\u009c\u0001\u0000\u0000\u0000\u00ac\u009d\u0001\u0000\u0000"+ - "\u0000\u00ac\u009e\u0001\u0000\u0000\u0000\u00ac\u009f\u0001\u0000\u0000"+ - "\u0000\u00ac\u00a0\u0001\u0000\u0000\u0000\u00ac\u00a1\u0001\u0000\u0000"+ - "\u0000\u00ac\u00a2\u0001\u0000\u0000\u0000\u00ac\u00a3\u0001\u0000\u0000"+ - "\u0000\u00ac\u00a4\u0001\u0000\u0000\u0000\u00ac\u00a5\u0001\u0000\u0000"+ - "\u0000\u00ac\u00a6\u0001\u0000\u0000\u0000\u00ac\u00a8\u0001\u0000\u0000"+ - "\u0000\u00ac\u00aa\u0001\u0000\u0000\u0000\u00ad\u0007\u0001\u0000\u0000"+ - "\u0000\u00ae\u00af\u0005\u0010\u0000\u0000\u00af\u00b0\u0003\n\u0005\u0000"+ - "\u00b0\t\u0001\u0000\u0000\u0000\u00b1\u00b2\u0006\u0005\uffff\uffff\u0000"+ - "\u00b2\u00b3\u00051\u0000\u0000\u00b3\u00cf\u0003\n\u0005\b\u00b4\u00cf"+ - "\u0003\u0010\b\u0000\u00b5\u00cf\u0003\f\u0006\u0000\u00b6\u00b8\u0003"+ - "\u0010\b\u0000\u00b7\u00b9\u00051\u0000\u0000\u00b8\u00b7\u0001\u0000"+ - "\u0000\u0000\u00b8\u00b9\u0001\u0000\u0000\u0000\u00b9\u00ba\u0001\u0000"+ - "\u0000\u0000\u00ba\u00bb\u0005,\u0000\u0000\u00bb\u00bc\u00050\u0000\u0000"+ - "\u00bc\u00c1\u0003\u0010\b\u0000\u00bd\u00be\u0005\'\u0000\u0000\u00be"+ - "\u00c0\u0003\u0010\b\u0000\u00bf\u00bd\u0001\u0000\u0000\u0000\u00c0\u00c3"+ - "\u0001\u0000\u0000\u0000\u00c1\u00bf\u0001\u0000\u0000\u0000\u00c1\u00c2"+ - "\u0001\u0000\u0000\u0000\u00c2\u00c4\u0001\u0000\u0000\u0000\u00c3\u00c1"+ - "\u0001\u0000\u0000\u0000\u00c4\u00c5\u00057\u0000\u0000\u00c5\u00cf\u0001"+ - "\u0000\u0000\u0000\u00c6\u00c7\u0003\u0010\b\u0000\u00c7\u00c9\u0005-"+ - "\u0000\u0000\u00c8\u00ca\u00051\u0000\u0000\u00c9\u00c8\u0001\u0000\u0000"+ - "\u0000\u00c9\u00ca\u0001\u0000\u0000\u0000\u00ca\u00cb\u0001\u0000\u0000"+ - "\u0000\u00cb\u00cc\u00052\u0000\u0000\u00cc\u00cf\u0001\u0000\u0000\u0000"+ - "\u00cd\u00cf\u0003\u000e\u0007\u0000\u00ce\u00b1\u0001\u0000\u0000\u0000"+ - "\u00ce\u00b4\u0001\u0000\u0000\u0000\u00ce\u00b5\u0001\u0000\u0000\u0000"+ - "\u00ce\u00b6\u0001\u0000\u0000\u0000\u00ce\u00c6\u0001\u0000\u0000\u0000"+ - "\u00ce\u00cd\u0001\u0000\u0000\u0000\u00cf\u00d8\u0001\u0000\u0000\u0000"+ - "\u00d0\u00d1\n\u0005\u0000\u0000\u00d1\u00d2\u0005\"\u0000\u0000\u00d2"+ - "\u00d7\u0003\n\u0005\u0006\u00d3\u00d4\n\u0004\u0000\u0000\u00d4\u00d5"+ - "\u00054\u0000\u0000\u00d5\u00d7\u0003\n\u0005\u0005\u00d6\u00d0\u0001"+ - "\u0000\u0000\u0000\u00d6\u00d3\u0001\u0000\u0000\u0000\u00d7\u00da\u0001"+ - "\u0000\u0000\u0000\u00d8\u00d6\u0001\u0000\u0000\u0000\u00d8\u00d9\u0001"+ - "\u0000\u0000\u0000\u00d9\u000b\u0001\u0000\u0000\u0000\u00da\u00d8\u0001"+ - "\u0000\u0000\u0000\u00db\u00dd\u0003\u0010\b\u0000\u00dc\u00de\u00051"+ - "\u0000\u0000\u00dd\u00dc\u0001\u0000\u0000\u0000\u00dd\u00de\u0001\u0000"+ - "\u0000\u0000\u00de\u00df\u0001\u0000\u0000\u0000\u00df\u00e0\u0005/\u0000"+ - "\u0000\u00e0\u00e1\u0003j5\u0000\u00e1\u00ea\u0001\u0000\u0000\u0000\u00e2"+ - "\u00e4\u0003\u0010\b\u0000\u00e3\u00e5\u00051\u0000\u0000\u00e4\u00e3"+ - "\u0001\u0000\u0000\u0000\u00e4\u00e5\u0001\u0000\u0000\u0000\u00e5\u00e6"+ - "\u0001\u0000\u0000\u0000\u00e6\u00e7\u00056\u0000\u0000\u00e7\u00e8\u0003"+ - "j5\u0000\u00e8\u00ea\u0001\u0000\u0000\u0000\u00e9\u00db\u0001\u0000\u0000"+ - "\u0000\u00e9\u00e2\u0001\u0000\u0000\u0000\u00ea\r\u0001\u0000\u0000\u0000"+ - "\u00eb\u00ee\u0003:\u001d\u0000\u00ec\u00ed\u0005%\u0000\u0000\u00ed\u00ef"+ - "\u0003\u001a\r\u0000\u00ee\u00ec\u0001\u0000\u0000\u0000\u00ee\u00ef\u0001"+ - "\u0000\u0000\u0000\u00ef\u00f0\u0001\u0000\u0000\u0000\u00f0\u00f1\u0005"+ - "&\u0000\u0000\u00f1\u00f2\u0003D\"\u0000\u00f2\u000f\u0001\u0000\u0000"+ - "\u0000\u00f3\u00f9\u0003\u0012\t\u0000\u00f4\u00f5\u0003\u0012\t\u0000"+ - "\u00f5\u00f6\u0003l6\u0000\u00f6\u00f7\u0003\u0012\t\u0000\u00f7\u00f9"+ - "\u0001\u0000\u0000\u0000\u00f8\u00f3\u0001\u0000\u0000\u0000\u00f8\u00f4"+ - "\u0001\u0000\u0000\u0000\u00f9\u0011\u0001\u0000\u0000\u0000\u00fa\u00fb"+ - "\u0006\t\uffff\uffff\u0000\u00fb\u00ff\u0003\u0014\n\u0000\u00fc\u00fd"+ - "\u0007\u0000\u0000\u0000\u00fd\u00ff\u0003\u0012\t\u0003\u00fe\u00fa\u0001"+ - "\u0000\u0000\u0000\u00fe\u00fc\u0001\u0000\u0000\u0000\u00ff\u0108\u0001"+ - "\u0000\u0000\u0000\u0100\u0101\n\u0002\u0000\u0000\u0101\u0102\u0007\u0001"+ - "\u0000\u0000\u0102\u0107\u0003\u0012\t\u0003\u0103\u0104\n\u0001\u0000"+ - "\u0000\u0104\u0105\u0007\u0000\u0000\u0000\u0105\u0107\u0003\u0012\t\u0002"+ - "\u0106\u0100\u0001\u0000\u0000\u0000\u0106\u0103\u0001\u0000\u0000\u0000"+ - "\u0107\u010a\u0001\u0000\u0000\u0000\u0108\u0106\u0001\u0000\u0000\u0000"+ - "\u0108\u0109\u0001\u0000\u0000\u0000\u0109\u0013\u0001\u0000\u0000\u0000"+ - "\u010a\u0108\u0001\u0000\u0000\u0000\u010b\u010c\u0006\n\uffff\uffff\u0000"+ - "\u010c\u0114\u0003D\"\u0000\u010d\u0114\u0003:\u001d\u0000\u010e\u0114"+ - "\u0003\u0016\u000b\u0000\u010f\u0110\u00050\u0000\u0000\u0110\u0111\u0003"+ - "\n\u0005\u0000\u0111\u0112\u00057\u0000\u0000\u0112\u0114\u0001\u0000"+ - "\u0000\u0000\u0113\u010b\u0001\u0000\u0000\u0000\u0113\u010d\u0001\u0000"+ - "\u0000\u0000\u0113\u010e\u0001\u0000\u0000\u0000\u0113\u010f\u0001\u0000"+ - "\u0000\u0000\u0114\u011a\u0001\u0000\u0000\u0000\u0115\u0116\n\u0001\u0000"+ - "\u0000\u0116\u0117\u0005%\u0000\u0000\u0117\u0119\u0003\u001a\r\u0000"+ - "\u0118\u0115\u0001\u0000\u0000\u0000\u0119\u011c\u0001\u0000\u0000\u0000"+ - "\u011a\u0118\u0001\u0000\u0000\u0000\u011a\u011b\u0001\u0000\u0000\u0000"+ - "\u011b\u0015\u0001\u0000\u0000\u0000\u011c\u011a\u0001\u0000\u0000\u0000"+ - "\u011d\u011e\u0003\u0018\f\u0000\u011e\u0128\u00050\u0000\u0000\u011f"+ - "\u0129\u0005B\u0000\u0000\u0120\u0125\u0003\n\u0005\u0000\u0121\u0122"+ - "\u0005\'\u0000\u0000\u0122\u0124\u0003\n\u0005\u0000\u0123\u0121\u0001"+ - "\u0000\u0000\u0000\u0124\u0127\u0001\u0000\u0000\u0000\u0125\u0123\u0001"+ - "\u0000\u0000\u0000\u0125\u0126\u0001\u0000\u0000\u0000\u0126\u0129\u0001"+ - "\u0000\u0000\u0000\u0127\u0125\u0001\u0000\u0000\u0000\u0128\u011f\u0001"+ - "\u0000\u0000\u0000\u0128\u0120\u0001\u0000\u0000\u0000\u0128\u0129\u0001"+ - "\u0000\u0000\u0000\u0129\u012a\u0001\u0000\u0000\u0000\u012a\u012b\u0005"+ - "7\u0000\u0000\u012b\u0017\u0001\u0000\u0000\u0000\u012c\u012d\u0003H$"+ - "\u0000\u012d\u0019\u0001\u0000\u0000\u0000\u012e\u012f\u0003@ \u0000\u012f"+ - "\u001b\u0001\u0000\u0000\u0000\u0130\u0131\u0005\f\u0000\u0000\u0131\u0132"+ - "\u0003\u001e\u000f\u0000\u0132\u001d\u0001\u0000\u0000\u0000\u0133\u0138"+ - "\u0003 \u0010\u0000\u0134\u0135\u0005\'\u0000\u0000\u0135\u0137\u0003"+ - " \u0010\u0000\u0136\u0134\u0001\u0000\u0000\u0000\u0137\u013a\u0001\u0000"+ - "\u0000\u0000\u0138\u0136\u0001\u0000\u0000\u0000\u0138\u0139\u0001\u0000"+ - "\u0000\u0000\u0139\u001f\u0001\u0000\u0000\u0000\u013a\u0138\u0001\u0000"+ - "\u0000\u0000\u013b\u013c\u0003:\u001d\u0000\u013c\u013d\u0005$\u0000\u0000"+ - "\u013d\u013f\u0001\u0000\u0000\u0000\u013e\u013b\u0001\u0000\u0000\u0000"+ - "\u013e\u013f\u0001\u0000\u0000\u0000\u013f\u0140\u0001\u0000\u0000\u0000"+ - "\u0140\u0141\u0003\n\u0005\u0000\u0141!\u0001\u0000\u0000\u0000\u0142"+ - "\u0143\u0005\u0006\u0000\u0000\u0143\u0148\u0003$\u0012\u0000\u0144\u0145"+ - "\u0005\'\u0000\u0000\u0145\u0147\u0003$\u0012\u0000\u0146\u0144\u0001"+ - "\u0000\u0000\u0000\u0147\u014a\u0001\u0000\u0000\u0000\u0148\u0146\u0001"+ - "\u0000\u0000\u0000\u0148\u0149\u0001\u0000\u0000\u0000\u0149\u014c\u0001"+ - "\u0000\u0000\u0000\u014a\u0148\u0001\u0000\u0000\u0000\u014b\u014d\u0003"+ - "*\u0015\u0000\u014c\u014b\u0001\u0000\u0000\u0000\u014c\u014d\u0001\u0000"+ - "\u0000\u0000\u014d#\u0001\u0000\u0000\u0000\u014e\u014f\u0003&\u0013\u0000"+ - "\u014f\u0150\u0005&\u0000\u0000\u0150\u0152\u0001\u0000\u0000\u0000\u0151"+ - "\u014e\u0001\u0000\u0000\u0000\u0151\u0152\u0001\u0000\u0000\u0000\u0152"+ - "\u0153\u0001\u0000\u0000\u0000\u0153\u0154\u0003(\u0014\u0000\u0154%\u0001"+ - "\u0000\u0000\u0000\u0155\u0156\u0005Q\u0000\u0000\u0156\'\u0001\u0000"+ - "\u0000\u0000\u0157\u0158\u0007\u0002\u0000\u0000\u0158)\u0001\u0000\u0000"+ - "\u0000\u0159\u015c\u0003,\u0016\u0000\u015a\u015c\u0003.\u0017\u0000\u015b"+ - "\u0159\u0001\u0000\u0000\u0000\u015b\u015a\u0001\u0000\u0000\u0000\u015c"+ - "+\u0001\u0000\u0000\u0000\u015d\u015e\u0005P\u0000\u0000\u015e\u0163\u0005"+ - "Q\u0000\u0000\u015f\u0160\u0005\'\u0000\u0000\u0160\u0162\u0005Q\u0000"+ - "\u0000\u0161\u015f\u0001\u0000\u0000\u0000\u0162\u0165\u0001\u0000\u0000"+ - "\u0000\u0163\u0161\u0001\u0000\u0000\u0000\u0163\u0164\u0001\u0000\u0000"+ - "\u0000\u0164-\u0001\u0000\u0000\u0000\u0165\u0163\u0001\u0000\u0000\u0000"+ - "\u0166\u0167\u0005F\u0000\u0000\u0167\u0168\u0003,\u0016\u0000\u0168\u0169"+ - "\u0005G\u0000\u0000\u0169/\u0001\u0000\u0000\u0000\u016a\u016b\u0005\u0013"+ - "\u0000\u0000\u016b\u0170\u0003$\u0012\u0000\u016c\u016d\u0005\'\u0000"+ - "\u0000\u016d\u016f\u0003$\u0012\u0000\u016e\u016c\u0001\u0000\u0000\u0000"+ - "\u016f\u0172\u0001\u0000\u0000\u0000\u0170\u016e\u0001\u0000\u0000\u0000"+ - "\u0170\u0171\u0001\u0000\u0000\u0000\u0171\u0174\u0001\u0000\u0000\u0000"+ - "\u0172\u0170\u0001\u0000\u0000\u0000\u0173\u0175\u00036\u001b\u0000\u0174"+ - "\u0173\u0001\u0000\u0000\u0000\u0174\u0175\u0001\u0000\u0000\u0000\u0175"+ - "\u0178\u0001\u0000\u0000\u0000\u0176\u0177\u0005!\u0000\u0000\u0177\u0179"+ - "\u0003\u001e\u000f\u0000\u0178\u0176\u0001\u0000\u0000\u0000\u0178\u0179"+ - "\u0001\u0000\u0000\u0000\u01791\u0001\u0000\u0000\u0000\u017a\u017b\u0005"+ - "\u0004\u0000\u0000\u017b\u017c\u0003\u001e\u000f\u0000\u017c3\u0001\u0000"+ - "\u0000\u0000\u017d\u017f\u0005\u000f\u0000\u0000\u017e\u0180\u00036\u001b"+ - "\u0000\u017f\u017e\u0001\u0000\u0000\u0000\u017f\u0180\u0001\u0000\u0000"+ - "\u0000\u0180\u0183\u0001\u0000\u0000\u0000\u0181\u0182\u0005!\u0000\u0000"+ - "\u0182\u0184\u0003\u001e\u000f\u0000\u0183\u0181\u0001\u0000\u0000\u0000"+ - "\u0183\u0184\u0001\u0000\u0000\u0000\u01845\u0001\u0000\u0000\u0000\u0185"+ - "\u018a\u00038\u001c\u0000\u0186\u0187\u0005\'\u0000\u0000\u0187\u0189"+ - "\u00038\u001c\u0000\u0188\u0186\u0001\u0000\u0000\u0000\u0189\u018c\u0001"+ - "\u0000\u0000\u0000\u018a\u0188\u0001\u0000\u0000\u0000\u018a\u018b\u0001"+ - "\u0000\u0000\u0000\u018b7\u0001\u0000\u0000\u0000\u018c\u018a\u0001\u0000"+ - "\u0000\u0000\u018d\u0190\u0003 \u0010\u0000\u018e\u018f\u0005\u0010\u0000"+ - "\u0000\u018f\u0191\u0003\n\u0005\u0000\u0190\u018e\u0001\u0000\u0000\u0000"+ - "\u0190\u0191\u0001\u0000\u0000\u0000\u01919\u0001\u0000\u0000\u0000\u0192"+ - "\u0197\u0003H$\u0000\u0193\u0194\u0005)\u0000\u0000\u0194\u0196\u0003"+ - "H$\u0000\u0195\u0193\u0001\u0000\u0000\u0000\u0196\u0199\u0001\u0000\u0000"+ - "\u0000\u0197\u0195\u0001\u0000\u0000\u0000\u0197\u0198\u0001\u0000\u0000"+ - "\u0000\u0198;\u0001\u0000\u0000\u0000\u0199\u0197\u0001\u0000\u0000\u0000"+ - "\u019a\u019f\u0003B!\u0000\u019b\u019c\u0005)\u0000\u0000\u019c\u019e"+ - "\u0003B!\u0000\u019d\u019b\u0001\u0000\u0000\u0000\u019e\u01a1\u0001\u0000"+ - "\u0000\u0000\u019f\u019d\u0001\u0000\u0000\u0000\u019f\u01a0\u0001\u0000"+ - "\u0000\u0000\u01a0=\u0001\u0000\u0000\u0000\u01a1\u019f\u0001\u0000\u0000"+ - "\u0000\u01a2\u01a7\u0003<\u001e\u0000\u01a3\u01a4\u0005\'\u0000\u0000"+ - "\u01a4\u01a6\u0003<\u001e\u0000\u01a5\u01a3\u0001\u0000\u0000\u0000\u01a6"+ - "\u01a9\u0001\u0000\u0000\u0000\u01a7\u01a5\u0001\u0000\u0000\u0000\u01a7"+ - "\u01a8\u0001\u0000\u0000\u0000\u01a8?\u0001\u0000\u0000\u0000\u01a9\u01a7"+ - "\u0001\u0000\u0000\u0000\u01aa\u01ab\u0007\u0003\u0000\u0000\u01abA\u0001"+ - "\u0000\u0000\u0000\u01ac\u01b0\u0005U\u0000\u0000\u01ad\u01ae\u0004!\n"+ - "\u0000\u01ae\u01b0\u0003F#\u0000\u01af\u01ac\u0001\u0000\u0000\u0000\u01af"+ - "\u01ad\u0001\u0000\u0000\u0000\u01b0C\u0001\u0000\u0000\u0000\u01b1\u01dc"+ - "\u00052\u0000\u0000\u01b2\u01b3\u0003h4\u0000\u01b3\u01b4\u0005H\u0000"+ - "\u0000\u01b4\u01dc\u0001\u0000\u0000\u0000\u01b5\u01dc\u0003f3\u0000\u01b6"+ - "\u01dc\u0003h4\u0000\u01b7\u01dc\u0003b1\u0000\u01b8\u01dc\u0003F#\u0000"+ - "\u01b9\u01dc\u0003j5\u0000\u01ba\u01bb\u0005F\u0000\u0000\u01bb\u01c0"+ - "\u0003d2\u0000\u01bc\u01bd\u0005\'\u0000\u0000\u01bd\u01bf\u0003d2\u0000"+ - "\u01be\u01bc\u0001\u0000\u0000\u0000\u01bf\u01c2\u0001\u0000\u0000\u0000"+ - "\u01c0\u01be\u0001\u0000\u0000\u0000\u01c0\u01c1\u0001\u0000\u0000\u0000"+ - "\u01c1\u01c3\u0001\u0000\u0000\u0000\u01c2\u01c0\u0001\u0000\u0000\u0000"+ - "\u01c3\u01c4\u0005G\u0000\u0000\u01c4\u01dc\u0001\u0000\u0000\u0000\u01c5"+ - "\u01c6\u0005F\u0000\u0000\u01c6\u01cb\u0003b1\u0000\u01c7\u01c8\u0005"+ - "\'\u0000\u0000\u01c8\u01ca\u0003b1\u0000\u01c9\u01c7\u0001\u0000\u0000"+ - "\u0000\u01ca\u01cd\u0001\u0000\u0000\u0000\u01cb\u01c9\u0001\u0000\u0000"+ - "\u0000\u01cb\u01cc\u0001\u0000\u0000\u0000\u01cc\u01ce\u0001\u0000\u0000"+ - "\u0000\u01cd\u01cb\u0001\u0000\u0000\u0000\u01ce\u01cf\u0005G\u0000\u0000"+ - "\u01cf\u01dc\u0001\u0000\u0000\u0000\u01d0\u01d1\u0005F\u0000\u0000\u01d1"+ - "\u01d6\u0003j5\u0000\u01d2\u01d3\u0005\'\u0000\u0000\u01d3\u01d5\u0003"+ - "j5\u0000\u01d4\u01d2\u0001\u0000\u0000\u0000\u01d5\u01d8\u0001\u0000\u0000"+ - "\u0000\u01d6\u01d4\u0001\u0000\u0000\u0000\u01d6\u01d7\u0001\u0000\u0000"+ - "\u0000\u01d7\u01d9\u0001\u0000\u0000\u0000\u01d8\u01d6\u0001\u0000\u0000"+ - "\u0000\u01d9\u01da\u0005G\u0000\u0000\u01da\u01dc\u0001\u0000\u0000\u0000"+ - "\u01db\u01b1\u0001\u0000\u0000\u0000\u01db\u01b2\u0001\u0000\u0000\u0000"+ - "\u01db\u01b5\u0001\u0000\u0000\u0000\u01db\u01b6\u0001\u0000\u0000\u0000"+ - "\u01db\u01b7\u0001\u0000\u0000\u0000\u01db\u01b8\u0001\u0000\u0000\u0000"+ - "\u01db\u01b9\u0001\u0000\u0000\u0000\u01db\u01ba\u0001\u0000\u0000\u0000"+ - "\u01db\u01c5\u0001\u0000\u0000\u0000\u01db\u01d0\u0001\u0000\u0000\u0000"+ - "\u01dcE\u0001\u0000\u0000\u0000\u01dd\u01e0\u00055\u0000\u0000\u01de\u01e0"+ - "\u0005E\u0000\u0000\u01df\u01dd\u0001\u0000\u0000\u0000\u01df\u01de\u0001"+ - "\u0000\u0000\u0000\u01e0G\u0001\u0000\u0000\u0000\u01e1\u01e5\u0003@ "+ - "\u0000\u01e2\u01e3\u0004$\u000b\u0000\u01e3\u01e5\u0003F#\u0000\u01e4"+ - "\u01e1\u0001\u0000\u0000\u0000\u01e4\u01e2\u0001\u0000\u0000\u0000\u01e5"+ - "I\u0001\u0000\u0000\u0000\u01e6\u01e7\u0005\t\u0000\u0000\u01e7\u01e8"+ - "\u0005\u001f\u0000\u0000\u01e8K\u0001\u0000\u0000\u0000\u01e9\u01ea\u0005"+ - "\u000e\u0000\u0000\u01ea\u01ef\u0003N\'\u0000\u01eb\u01ec\u0005\'\u0000"+ - "\u0000\u01ec\u01ee\u0003N\'\u0000\u01ed\u01eb\u0001\u0000\u0000\u0000"+ - "\u01ee\u01f1\u0001\u0000\u0000\u0000\u01ef\u01ed\u0001\u0000\u0000\u0000"+ - "\u01ef\u01f0\u0001\u0000\u0000\u0000\u01f0M\u0001\u0000\u0000\u0000\u01f1"+ - "\u01ef\u0001\u0000\u0000\u0000\u01f2\u01f4\u0003\n\u0005\u0000\u01f3\u01f5"+ - "\u0007\u0004\u0000\u0000\u01f4\u01f3\u0001\u0000\u0000\u0000\u01f4\u01f5"+ - "\u0001\u0000\u0000\u0000\u01f5\u01f8\u0001\u0000\u0000\u0000\u01f6\u01f7"+ - "\u00053\u0000\u0000\u01f7\u01f9\u0007\u0005\u0000\u0000\u01f8\u01f6\u0001"+ - "\u0000\u0000\u0000\u01f8\u01f9\u0001\u0000\u0000\u0000\u01f9O\u0001\u0000"+ - "\u0000\u0000\u01fa\u01fb\u0005\b\u0000\u0000\u01fb\u01fc\u0003>\u001f"+ - "\u0000\u01fcQ\u0001\u0000\u0000\u0000\u01fd\u01fe\u0005\u0002\u0000\u0000"+ - "\u01fe\u01ff\u0003>\u001f\u0000\u01ffS\u0001\u0000\u0000\u0000\u0200\u0201"+ - "\u0005\u000b\u0000\u0000\u0201\u0206\u0003V+\u0000\u0202\u0203\u0005\'"+ - "\u0000\u0000\u0203\u0205\u0003V+\u0000\u0204\u0202\u0001\u0000\u0000\u0000"+ - "\u0205\u0208\u0001\u0000\u0000\u0000\u0206\u0204\u0001\u0000\u0000\u0000"+ - "\u0206\u0207\u0001\u0000\u0000\u0000\u0207U\u0001\u0000\u0000\u0000\u0208"+ - "\u0206\u0001\u0000\u0000\u0000\u0209\u020a\u0003<\u001e\u0000\u020a\u020b"+ - "\u0005Y\u0000\u0000\u020b\u020c\u0003<\u001e\u0000\u020cW\u0001\u0000"+ - "\u0000\u0000\u020d\u020e\u0005\u0001\u0000\u0000\u020e\u020f\u0003\u0014"+ - "\n\u0000\u020f\u0211\u0003j5\u0000\u0210\u0212\u0003^/\u0000\u0211\u0210"+ - "\u0001\u0000\u0000\u0000\u0211\u0212\u0001\u0000\u0000\u0000\u0212Y\u0001"+ - "\u0000\u0000\u0000\u0213\u0214\u0005\u0007\u0000\u0000\u0214\u0215\u0003"+ - "\u0014\n\u0000\u0215\u0216\u0003j5\u0000\u0216[\u0001\u0000\u0000\u0000"+ - "\u0217\u0218\u0005\n\u0000\u0000\u0218\u0219\u0003:\u001d\u0000\u0219"+ - "]\u0001\u0000\u0000\u0000\u021a\u021f\u0003`0\u0000\u021b\u021c\u0005"+ - "\'\u0000\u0000\u021c\u021e\u0003`0\u0000\u021d\u021b\u0001\u0000\u0000"+ - "\u0000\u021e\u0221\u0001\u0000\u0000\u0000\u021f\u021d\u0001\u0000\u0000"+ - "\u0000\u021f\u0220\u0001\u0000\u0000\u0000\u0220_\u0001\u0000\u0000\u0000"+ - "\u0221\u021f\u0001\u0000\u0000\u0000\u0222\u0223\u0003@ \u0000\u0223\u0224"+ - "\u0005$\u0000\u0000\u0224\u0225\u0003D\"\u0000\u0225a\u0001\u0000\u0000"+ - "\u0000\u0226\u0227\u0007\u0006\u0000\u0000\u0227c\u0001\u0000\u0000\u0000"+ - "\u0228\u022b\u0003f3\u0000\u0229\u022b\u0003h4\u0000\u022a\u0228\u0001"+ - "\u0000\u0000\u0000\u022a\u0229\u0001\u0000\u0000\u0000\u022be\u0001\u0000"+ - "\u0000\u0000\u022c\u022e\u0007\u0000\u0000\u0000\u022d\u022c\u0001\u0000"+ - "\u0000\u0000\u022d\u022e\u0001\u0000\u0000\u0000\u022e\u022f\u0001\u0000"+ - "\u0000\u0000\u022f\u0230\u0005 \u0000\u0000\u0230g\u0001\u0000\u0000\u0000"+ - "\u0231\u0233\u0007\u0000\u0000\u0000\u0232\u0231\u0001\u0000\u0000\u0000"+ - "\u0232\u0233\u0001\u0000\u0000\u0000\u0233\u0234\u0001\u0000\u0000\u0000"+ - "\u0234\u0235\u0005\u001f\u0000\u0000\u0235i\u0001\u0000\u0000\u0000\u0236"+ - "\u0237\u0005\u001e\u0000\u0000\u0237k\u0001\u0000\u0000\u0000\u0238\u0239"+ - "\u0007\u0007\u0000\u0000\u0239m\u0001\u0000\u0000\u0000\u023a\u023b\u0005"+ - "\u0005\u0000\u0000\u023b\u023c\u0003p8\u0000\u023co\u0001\u0000\u0000"+ - "\u0000\u023d\u023e\u0005F\u0000\u0000\u023e\u023f\u0003\u0002\u0001\u0000"+ - "\u023f\u0240\u0005G\u0000\u0000\u0240q\u0001\u0000\u0000\u0000\u0241\u0242"+ - "\u0005\r\u0000\u0000\u0242\u0243\u0005i\u0000\u0000\u0243s\u0001\u0000"+ - "\u0000\u0000\u0244\u0245\u0005\u0003\u0000\u0000\u0245\u0248\u0005_\u0000"+ - "\u0000\u0246\u0247\u0005]\u0000\u0000\u0247\u0249\u0003<\u001e\u0000\u0248"+ - "\u0246\u0001\u0000\u0000\u0000\u0248\u0249\u0001\u0000\u0000\u0000\u0249"+ - "\u0253\u0001\u0000\u0000\u0000\u024a\u024b\u0005^\u0000\u0000\u024b\u0250"+ - "\u0003v;\u0000\u024c\u024d\u0005\'\u0000\u0000\u024d\u024f\u0003v;\u0000"+ - "\u024e\u024c\u0001\u0000\u0000\u0000\u024f\u0252\u0001\u0000\u0000\u0000"+ - "\u0250\u024e\u0001\u0000\u0000\u0000\u0250\u0251\u0001\u0000\u0000\u0000"+ - "\u0251\u0254\u0001\u0000\u0000\u0000\u0252\u0250\u0001\u0000\u0000\u0000"+ - "\u0253\u024a\u0001\u0000\u0000\u0000\u0253\u0254\u0001\u0000\u0000\u0000"+ - "\u0254u\u0001\u0000\u0000\u0000\u0255\u0256\u0003<\u001e\u0000\u0256\u0257"+ - "\u0005$\u0000\u0000\u0257\u0259\u0001\u0000\u0000\u0000\u0258\u0255\u0001"+ - "\u0000\u0000\u0000\u0258\u0259\u0001\u0000\u0000\u0000\u0259\u025a\u0001"+ - "\u0000\u0000\u0000\u025a\u025b\u0003<\u001e\u0000\u025bw\u0001\u0000\u0000"+ - "\u0000\u025c\u025d\u0005\u0012\u0000\u0000\u025d\u025e\u0003$\u0012\u0000"+ - "\u025e\u025f\u0005]\u0000\u0000\u025f\u0260\u0003>\u001f\u0000\u0260y"+ - "\u0001\u0000\u0000\u0000\u0261\u0262\u0005\u0011\u0000\u0000\u0262\u0265"+ - "\u00036\u001b\u0000\u0263\u0264\u0005!\u0000\u0000\u0264\u0266\u0003\u001e"+ - "\u000f\u0000\u0265\u0263\u0001\u0000\u0000\u0000\u0265\u0266\u0001\u0000"+ - "\u0000\u0000\u0266{\u0001\u0000\u0000\u0000\u0267\u0269\u0007\b\u0000"+ - "\u0000\u0268\u0267\u0001\u0000\u0000\u0000\u0268\u0269\u0001\u0000\u0000"+ - "\u0000\u0269\u026a\u0001\u0000\u0000\u0000\u026a\u026b\u0005\u0014\u0000"+ - "\u0000\u026b\u026c\u0003~?\u0000\u026c\u026d\u0003\u0080@\u0000\u026d"+ - "}\u0001\u0000\u0000\u0000\u026e\u0271\u0003@ \u0000\u026f\u0270\u0005"+ - "Y\u0000\u0000\u0270\u0272\u0003@ \u0000\u0271\u026f\u0001\u0000\u0000"+ - "\u0000\u0271\u0272\u0001\u0000\u0000\u0000\u0272\u007f\u0001\u0000\u0000"+ - "\u0000\u0273\u0274\u0005]\u0000\u0000\u0274\u0279\u0003\u0082A\u0000\u0275"+ - "\u0276\u0005\'\u0000\u0000\u0276\u0278\u0003\u0082A\u0000\u0277\u0275"+ - "\u0001\u0000\u0000\u0000\u0278\u027b\u0001\u0000\u0000\u0000\u0279\u0277"+ - "\u0001\u0000\u0000\u0000\u0279\u027a\u0001\u0000\u0000\u0000\u027a\u0081"+ - "\u0001\u0000\u0000\u0000\u027b\u0279\u0001\u0000\u0000\u0000\u027c\u027d"+ - "\u0003\u0010\b\u0000\u027d\u0083\u0001\u0000\u0000\u0000>\u008f\u0098"+ - "\u00ac\u00b8\u00c1\u00c9\u00ce\u00d6\u00d8\u00dd\u00e4\u00e9\u00ee\u00f8"+ - "\u00fe\u0106\u0108\u0113\u011a\u0125\u0128\u0138\u013e\u0148\u014c\u0151"+ - "\u015b\u0163\u0170\u0174\u0178\u017f\u0183\u018a\u0190\u0197\u019f\u01a7"+ - "\u01af\u01c0\u01cb\u01d6\u01db\u01df\u01e4\u01ef\u01f4\u01f8\u0206\u0211"+ - "\u021f\u022a\u022d\u0232\u0248\u0250\u0253\u0258\u0265\u0268\u0271\u0279"; + ";\u0002<\u0007<\u0002=\u0007=\u0002>\u0007>\u0002?\u0007?\u0001\u0000"+ + "\u0001\u0000\u0001\u0000\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"+ + "\u0001\u0001\u0001\u0001\u0005\u0001\u008a\b\u0001\n\u0001\f\u0001\u008d"+ + "\t\u0001\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001"+ + "\u0002\u0003\u0002\u0095\b\u0002\u0001\u0003\u0001\u0003\u0001\u0003\u0001"+ + "\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001"+ + "\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001"+ + "\u0003\u0001\u0003\u0001\u0003\u0003\u0003\u00a9\b\u0003\u0001\u0004\u0001"+ + "\u0004\u0001\u0004\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+ + "\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00b5\b\u0005\u0001\u0005\u0001"+ + "\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0005\u0005\u00bc\b\u0005\n"+ + "\u0005\f\u0005\u00bf\t\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+ + "\u0005\u0001\u0005\u0003\u0005\u00c6\b\u0005\u0001\u0005\u0001\u0005\u0001"+ + "\u0005\u0003\u0005\u00cb\b\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+ + "\u0005\u0001\u0005\u0001\u0005\u0005\u0005\u00d3\b\u0005\n\u0005\f\u0005"+ + "\u00d6\t\u0005\u0001\u0006\u0001\u0006\u0003\u0006\u00da\b\u0006\u0001"+ + "\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0003\u0006\u00e1"+ + "\b\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0003\u0006\u00e6\b\u0006"+ + "\u0001\u0007\u0001\u0007\u0001\u0007\u0003\u0007\u00eb\b\u0007\u0001\u0007"+ + "\u0001\u0007\u0001\u0007\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0003"+ + "\b\u00f5\b\b\u0001\t\u0001\t\u0001\t\u0001\t\u0003\t\u00fb\b\t\u0001\t"+ + "\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0005\t\u0103\b\t\n\t\f\t\u0106"+ + "\t\t\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0003"+ + "\n\u0110\b\n\u0001\n\u0001\n\u0001\n\u0005\n\u0115\b\n\n\n\f\n\u0118\t"+ + "\n\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b"+ + "\u0005\u000b\u0120\b\u000b\n\u000b\f\u000b\u0123\t\u000b\u0003\u000b\u0125"+ + "\b\u000b\u0001\u000b\u0001\u000b\u0001\f\u0001\f\u0001\r\u0001\r\u0001"+ + "\u000e\u0001\u000e\u0001\u000e\u0001\u000f\u0001\u000f\u0001\u000f\u0005"+ + "\u000f\u0133\b\u000f\n\u000f\f\u000f\u0136\t\u000f\u0001\u0010\u0001\u0010"+ + "\u0001\u0010\u0003\u0010\u013b\b\u0010\u0001\u0010\u0001\u0010\u0001\u0011"+ + "\u0001\u0011\u0001\u0011\u0001\u0011\u0005\u0011\u0143\b\u0011\n\u0011"+ + "\f\u0011\u0146\t\u0011\u0001\u0011\u0003\u0011\u0149\b\u0011\u0001\u0012"+ + "\u0001\u0012\u0001\u0012\u0003\u0012\u014e\b\u0012\u0001\u0012\u0001\u0012"+ + "\u0001\u0013\u0001\u0013\u0001\u0014\u0001\u0014\u0001\u0015\u0001\u0015"+ + "\u0001\u0015\u0001\u0015\u0005\u0015\u015a\b\u0015\n\u0015\f\u0015\u015d"+ + "\t\u0015\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0016\u0005\u0016\u0163"+ + "\b\u0016\n\u0016\f\u0016\u0166\t\u0016\u0001\u0016\u0003\u0016\u0169\b"+ + "\u0016\u0001\u0016\u0001\u0016\u0003\u0016\u016d\b\u0016\u0001\u0017\u0001"+ + "\u0017\u0001\u0017\u0001\u0018\u0001\u0018\u0003\u0018\u0174\b\u0018\u0001"+ + "\u0018\u0001\u0018\u0003\u0018\u0178\b\u0018\u0001\u0019\u0001\u0019\u0001"+ + "\u0019\u0005\u0019\u017d\b\u0019\n\u0019\f\u0019\u0180\t\u0019\u0001\u001a"+ + "\u0001\u001a\u0001\u001a\u0003\u001a\u0185\b\u001a\u0001\u001b\u0001\u001b"+ + "\u0001\u001b\u0005\u001b\u018a\b\u001b\n\u001b\f\u001b\u018d\t\u001b\u0001"+ + "\u001c\u0001\u001c\u0001\u001c\u0005\u001c\u0192\b\u001c\n\u001c\f\u001c"+ + "\u0195\t\u001c\u0001\u001d\u0001\u001d\u0001\u001d\u0005\u001d\u019a\b"+ + "\u001d\n\u001d\f\u001d\u019d\t\u001d\u0001\u001e\u0001\u001e\u0001\u001f"+ + "\u0001\u001f\u0001\u001f\u0003\u001f\u01a4\b\u001f\u0001 \u0001 \u0001"+ + " \u0001 \u0001 \u0001 \u0001 \u0001 \u0001 \u0001 \u0001 \u0001 \u0001"+ + " \u0005 \u01b3\b \n \f \u01b6\t \u0001 \u0001 \u0001 \u0001 \u0001 \u0001"+ + " \u0005 \u01be\b \n \f \u01c1\t \u0001 \u0001 \u0001 \u0001 \u0001 \u0001"+ + " \u0005 \u01c9\b \n \f \u01cc\t \u0001 \u0001 \u0003 \u01d0\b \u0001!"+ + "\u0001!\u0003!\u01d4\b!\u0001\"\u0001\"\u0001\"\u0003\"\u01d9\b\"\u0001"+ + "#\u0001#\u0001#\u0001$\u0001$\u0001$\u0001$\u0005$\u01e2\b$\n$\f$\u01e5"+ + "\t$\u0001%\u0001%\u0003%\u01e9\b%\u0001%\u0001%\u0003%\u01ed\b%\u0001"+ + "&\u0001&\u0001&\u0001\'\u0001\'\u0001\'\u0001(\u0001(\u0001(\u0001(\u0005"+ + "(\u01f9\b(\n(\f(\u01fc\t(\u0001)\u0001)\u0001)\u0001)\u0001*\u0001*\u0001"+ + "*\u0001*\u0003*\u0206\b*\u0001+\u0001+\u0001+\u0001+\u0001,\u0001,\u0001"+ + ",\u0001-\u0001-\u0001-\u0005-\u0212\b-\n-\f-\u0215\t-\u0001.\u0001.\u0001"+ + ".\u0001.\u0001/\u0001/\u00010\u00010\u00030\u021f\b0\u00011\u00031\u0222"+ + "\b1\u00011\u00011\u00012\u00032\u0227\b2\u00012\u00012\u00013\u00013\u0001"+ + "4\u00014\u00015\u00015\u00015\u00016\u00016\u00016\u00016\u00017\u0001"+ + "7\u00017\u00018\u00018\u00018\u00018\u00038\u023d\b8\u00018\u00018\u0001"+ + "8\u00018\u00058\u0243\b8\n8\f8\u0246\t8\u00038\u0248\b8\u00019\u00019"+ + "\u00019\u00039\u024d\b9\u00019\u00019\u0001:\u0001:\u0001:\u0001:\u0001"+ + ":\u0001;\u0001;\u0001;\u0001;\u0003;\u025a\b;\u0001<\u0003<\u025d\b<\u0001"+ + "<\u0001<\u0001<\u0001<\u0001=\u0001=\u0001=\u0003=\u0266\b=\u0001>\u0001"+ + ">\u0001>\u0001>\u0005>\u026c\b>\n>\f>\u026f\t>\u0001?\u0001?\u0001?\u0000"+ + "\u0004\u0002\n\u0012\u0014@\u0000\u0002\u0004\u0006\b\n\f\u000e\u0010"+ + "\u0012\u0014\u0016\u0018\u001a\u001c\u001e \"$&(*,.02468:<>@BDFHJLNPR"+ + "TVXZ\\^`bdfhjlnprtvxz|~\u0000\t\u0001\u0000@A\u0001\u0000BD\u0002\u0000"+ + "\u001e\u001eQQ\u0001\u0000HI\u0002\u0000##((\u0002\u0000++..\u0002\u0000"+ + "**88\u0002\u000099;?\u0001\u0000\u0016\u0018\u028e\u0000\u0080\u0001\u0000"+ + "\u0000\u0000\u0002\u0083\u0001\u0000\u0000\u0000\u0004\u0094\u0001\u0000"+ + "\u0000\u0000\u0006\u00a8\u0001\u0000\u0000\u0000\b\u00aa\u0001\u0000\u0000"+ + "\u0000\n\u00ca\u0001\u0000\u0000\u0000\f\u00e5\u0001\u0000\u0000\u0000"+ + "\u000e\u00e7\u0001\u0000\u0000\u0000\u0010\u00f4\u0001\u0000\u0000\u0000"+ + "\u0012\u00fa\u0001\u0000\u0000\u0000\u0014\u010f\u0001\u0000\u0000\u0000"+ + "\u0016\u0119\u0001\u0000\u0000\u0000\u0018\u0128\u0001\u0000\u0000\u0000"+ + "\u001a\u012a\u0001\u0000\u0000\u0000\u001c\u012c\u0001\u0000\u0000\u0000"+ + "\u001e\u012f\u0001\u0000\u0000\u0000 \u013a\u0001\u0000\u0000\u0000\""+ + "\u013e\u0001\u0000\u0000\u0000$\u014d\u0001\u0000\u0000\u0000&\u0151\u0001"+ + "\u0000\u0000\u0000(\u0153\u0001\u0000\u0000\u0000*\u0155\u0001\u0000\u0000"+ + "\u0000,\u015e\u0001\u0000\u0000\u0000.\u016e\u0001\u0000\u0000\u00000"+ + "\u0171\u0001\u0000\u0000\u00002\u0179\u0001\u0000\u0000\u00004\u0181\u0001"+ + "\u0000\u0000\u00006\u0186\u0001\u0000\u0000\u00008\u018e\u0001\u0000\u0000"+ + "\u0000:\u0196\u0001\u0000\u0000\u0000<\u019e\u0001\u0000\u0000\u0000>"+ + "\u01a3\u0001\u0000\u0000\u0000@\u01cf\u0001\u0000\u0000\u0000B\u01d3\u0001"+ + "\u0000\u0000\u0000D\u01d8\u0001\u0000\u0000\u0000F\u01da\u0001\u0000\u0000"+ + "\u0000H\u01dd\u0001\u0000\u0000\u0000J\u01e6\u0001\u0000\u0000\u0000L"+ + "\u01ee\u0001\u0000\u0000\u0000N\u01f1\u0001\u0000\u0000\u0000P\u01f4\u0001"+ + "\u0000\u0000\u0000R\u01fd\u0001\u0000\u0000\u0000T\u0201\u0001\u0000\u0000"+ + "\u0000V\u0207\u0001\u0000\u0000\u0000X\u020b\u0001\u0000\u0000\u0000Z"+ + "\u020e\u0001\u0000\u0000\u0000\\\u0216\u0001\u0000\u0000\u0000^\u021a"+ + "\u0001\u0000\u0000\u0000`\u021e\u0001\u0000\u0000\u0000b\u0221\u0001\u0000"+ + "\u0000\u0000d\u0226\u0001\u0000\u0000\u0000f\u022a\u0001\u0000\u0000\u0000"+ + "h\u022c\u0001\u0000\u0000\u0000j\u022e\u0001\u0000\u0000\u0000l\u0231"+ + "\u0001\u0000\u0000\u0000n\u0235\u0001\u0000\u0000\u0000p\u0238\u0001\u0000"+ + "\u0000\u0000r\u024c\u0001\u0000\u0000\u0000t\u0250\u0001\u0000\u0000\u0000"+ + "v\u0255\u0001\u0000\u0000\u0000x\u025c\u0001\u0000\u0000\u0000z\u0262"+ + "\u0001\u0000\u0000\u0000|\u0267\u0001\u0000\u0000\u0000~\u0270\u0001\u0000"+ + "\u0000\u0000\u0080\u0081\u0003\u0002\u0001\u0000\u0081\u0082\u0005\u0000"+ + "\u0000\u0001\u0082\u0001\u0001\u0000\u0000\u0000\u0083\u0084\u0006\u0001"+ + "\uffff\uffff\u0000\u0084\u0085\u0003\u0004\u0002\u0000\u0085\u008b\u0001"+ + "\u0000\u0000\u0000\u0086\u0087\n\u0001\u0000\u0000\u0087\u0088\u0005\u001d"+ + "\u0000\u0000\u0088\u008a\u0003\u0006\u0003\u0000\u0089\u0086\u0001\u0000"+ + "\u0000\u0000\u008a\u008d\u0001\u0000\u0000\u0000\u008b\u0089\u0001\u0000"+ + "\u0000\u0000\u008b\u008c\u0001\u0000\u0000\u0000\u008c\u0003\u0001\u0000"+ + "\u0000\u0000\u008d\u008b\u0001\u0000\u0000\u0000\u008e\u0095\u0003j5\u0000"+ + "\u008f\u0095\u0003\"\u0011\u0000\u0090\u0095\u0003\u001c\u000e\u0000\u0091"+ + "\u0095\u0003n7\u0000\u0092\u0093\u0004\u0002\u0001\u0000\u0093\u0095\u0003"+ + ",\u0016\u0000\u0094\u008e\u0001\u0000\u0000\u0000\u0094\u008f\u0001\u0000"+ + "\u0000\u0000\u0094\u0090\u0001\u0000\u0000\u0000\u0094\u0091\u0001\u0000"+ + "\u0000\u0000\u0094\u0092\u0001\u0000\u0000\u0000\u0095\u0005\u0001\u0000"+ + "\u0000\u0000\u0096\u00a9\u0003.\u0017\u0000\u0097\u00a9\u0003\b\u0004"+ + "\u0000\u0098\u00a9\u0003L&\u0000\u0099\u00a9\u0003F#\u0000\u009a\u00a9"+ + "\u00030\u0018\u0000\u009b\u00a9\u0003H$\u0000\u009c\u00a9\u0003N\'\u0000"+ + "\u009d\u00a9\u0003P(\u0000\u009e\u00a9\u0003T*\u0000\u009f\u00a9\u0003"+ + "V+\u0000\u00a0\u00a9\u0003p8\u0000\u00a1\u00a9\u0003X,\u0000\u00a2\u00a3"+ + "\u0004\u0003\u0002\u0000\u00a3\u00a9\u0003v;\u0000\u00a4\u00a5\u0004\u0003"+ + "\u0003\u0000\u00a5\u00a9\u0003t:\u0000\u00a6\u00a7\u0004\u0003\u0004\u0000"+ + "\u00a7\u00a9\u0003x<\u0000\u00a8\u0096\u0001\u0000\u0000\u0000\u00a8\u0097"+ + "\u0001\u0000\u0000\u0000\u00a8\u0098\u0001\u0000\u0000\u0000\u00a8\u0099"+ + "\u0001\u0000\u0000\u0000\u00a8\u009a\u0001\u0000\u0000\u0000\u00a8\u009b"+ + "\u0001\u0000\u0000\u0000\u00a8\u009c\u0001\u0000\u0000\u0000\u00a8\u009d"+ + "\u0001\u0000\u0000\u0000\u00a8\u009e\u0001\u0000\u0000\u0000\u00a8\u009f"+ + "\u0001\u0000\u0000\u0000\u00a8\u00a0\u0001\u0000\u0000\u0000\u00a8\u00a1"+ + "\u0001\u0000\u0000\u0000\u00a8\u00a2\u0001\u0000\u0000\u0000\u00a8\u00a4"+ + "\u0001\u0000\u0000\u0000\u00a8\u00a6\u0001\u0000\u0000\u0000\u00a9\u0007"+ + "\u0001\u0000\u0000\u0000\u00aa\u00ab\u0005\u0010\u0000\u0000\u00ab\u00ac"+ + "\u0003\n\u0005\u0000\u00ac\t\u0001\u0000\u0000\u0000\u00ad\u00ae\u0006"+ + "\u0005\uffff\uffff\u0000\u00ae\u00af\u00051\u0000\u0000\u00af\u00cb\u0003"+ + "\n\u0005\b\u00b0\u00cb\u0003\u0010\b\u0000\u00b1\u00cb\u0003\f\u0006\u0000"+ + "\u00b2\u00b4\u0003\u0010\b\u0000\u00b3\u00b5\u00051\u0000\u0000\u00b4"+ + "\u00b3\u0001\u0000\u0000\u0000\u00b4\u00b5\u0001\u0000\u0000\u0000\u00b5"+ + "\u00b6\u0001\u0000\u0000\u0000\u00b6\u00b7\u0005,\u0000\u0000\u00b7\u00b8"+ + "\u00050\u0000\u0000\u00b8\u00bd\u0003\u0010\b\u0000\u00b9\u00ba\u0005"+ + "\'\u0000\u0000\u00ba\u00bc\u0003\u0010\b\u0000\u00bb\u00b9\u0001\u0000"+ + "\u0000\u0000\u00bc\u00bf\u0001\u0000\u0000\u0000\u00bd\u00bb\u0001\u0000"+ + "\u0000\u0000\u00bd\u00be\u0001\u0000\u0000\u0000\u00be\u00c0\u0001\u0000"+ + "\u0000\u0000\u00bf\u00bd\u0001\u0000\u0000\u0000\u00c0\u00c1\u00057\u0000"+ + "\u0000\u00c1\u00cb\u0001\u0000\u0000\u0000\u00c2\u00c3\u0003\u0010\b\u0000"+ + "\u00c3\u00c5\u0005-\u0000\u0000\u00c4\u00c6\u00051\u0000\u0000\u00c5\u00c4"+ + "\u0001\u0000\u0000\u0000\u00c5\u00c6\u0001\u0000\u0000\u0000\u00c6\u00c7"+ + "\u0001\u0000\u0000\u0000\u00c7\u00c8\u00052\u0000\u0000\u00c8\u00cb\u0001"+ + "\u0000\u0000\u0000\u00c9\u00cb\u0003\u000e\u0007\u0000\u00ca\u00ad\u0001"+ + "\u0000\u0000\u0000\u00ca\u00b0\u0001\u0000\u0000\u0000\u00ca\u00b1\u0001"+ + "\u0000\u0000\u0000\u00ca\u00b2\u0001\u0000\u0000\u0000\u00ca\u00c2\u0001"+ + "\u0000\u0000\u0000\u00ca\u00c9\u0001\u0000\u0000\u0000\u00cb\u00d4\u0001"+ + "\u0000\u0000\u0000\u00cc\u00cd\n\u0005\u0000\u0000\u00cd\u00ce\u0005\""+ + "\u0000\u0000\u00ce\u00d3\u0003\n\u0005\u0006\u00cf\u00d0\n\u0004\u0000"+ + "\u0000\u00d0\u00d1\u00054\u0000\u0000\u00d1\u00d3\u0003\n\u0005\u0005"+ + "\u00d2\u00cc\u0001\u0000\u0000\u0000\u00d2\u00cf\u0001\u0000\u0000\u0000"+ + "\u00d3\u00d6\u0001\u0000\u0000\u0000\u00d4\u00d2\u0001\u0000\u0000\u0000"+ + "\u00d4\u00d5\u0001\u0000\u0000\u0000\u00d5\u000b\u0001\u0000\u0000\u0000"+ + "\u00d6\u00d4\u0001\u0000\u0000\u0000\u00d7\u00d9\u0003\u0010\b\u0000\u00d8"+ + "\u00da\u00051\u0000\u0000\u00d9\u00d8\u0001\u0000\u0000\u0000\u00d9\u00da"+ + "\u0001\u0000\u0000\u0000\u00da\u00db\u0001\u0000\u0000\u0000\u00db\u00dc"+ + "\u0005/\u0000\u0000\u00dc\u00dd\u0003f3\u0000\u00dd\u00e6\u0001\u0000"+ + "\u0000\u0000\u00de\u00e0\u0003\u0010\b\u0000\u00df\u00e1\u00051\u0000"+ + "\u0000\u00e0\u00df\u0001\u0000\u0000\u0000\u00e0\u00e1\u0001\u0000\u0000"+ + "\u0000\u00e1\u00e2\u0001\u0000\u0000\u0000\u00e2\u00e3\u00056\u0000\u0000"+ + "\u00e3\u00e4\u0003f3\u0000\u00e4\u00e6\u0001\u0000\u0000\u0000\u00e5\u00d7"+ + "\u0001\u0000\u0000\u0000\u00e5\u00de\u0001\u0000\u0000\u0000\u00e6\r\u0001"+ + "\u0000\u0000\u0000\u00e7\u00ea\u00036\u001b\u0000\u00e8\u00e9\u0005%\u0000"+ + "\u0000\u00e9\u00eb\u0003\u001a\r\u0000\u00ea\u00e8\u0001\u0000\u0000\u0000"+ + "\u00ea\u00eb\u0001\u0000\u0000\u0000\u00eb\u00ec\u0001\u0000\u0000\u0000"+ + "\u00ec\u00ed\u0005&\u0000\u0000\u00ed\u00ee\u0003@ \u0000\u00ee\u000f"+ + "\u0001\u0000\u0000\u0000\u00ef\u00f5\u0003\u0012\t\u0000\u00f0\u00f1\u0003"+ + "\u0012\t\u0000\u00f1\u00f2\u0003h4\u0000\u00f2\u00f3\u0003\u0012\t\u0000"+ + "\u00f3\u00f5\u0001\u0000\u0000\u0000\u00f4\u00ef\u0001\u0000\u0000\u0000"+ + "\u00f4\u00f0\u0001\u0000\u0000\u0000\u00f5\u0011\u0001\u0000\u0000\u0000"+ + "\u00f6\u00f7\u0006\t\uffff\uffff\u0000\u00f7\u00fb\u0003\u0014\n\u0000"+ + "\u00f8\u00f9\u0007\u0000\u0000\u0000\u00f9\u00fb\u0003\u0012\t\u0003\u00fa"+ + "\u00f6\u0001\u0000\u0000\u0000\u00fa\u00f8\u0001\u0000\u0000\u0000\u00fb"+ + "\u0104\u0001\u0000\u0000\u0000\u00fc\u00fd\n\u0002\u0000\u0000\u00fd\u00fe"+ + "\u0007\u0001\u0000\u0000\u00fe\u0103\u0003\u0012\t\u0003\u00ff\u0100\n"+ + "\u0001\u0000\u0000\u0100\u0101\u0007\u0000\u0000\u0000\u0101\u0103\u0003"+ + "\u0012\t\u0002\u0102\u00fc\u0001\u0000\u0000\u0000\u0102\u00ff\u0001\u0000"+ + "\u0000\u0000\u0103\u0106\u0001\u0000\u0000\u0000\u0104\u0102\u0001\u0000"+ + "\u0000\u0000\u0104\u0105\u0001\u0000\u0000\u0000\u0105\u0013\u0001\u0000"+ + "\u0000\u0000\u0106\u0104\u0001\u0000\u0000\u0000\u0107\u0108\u0006\n\uffff"+ + "\uffff\u0000\u0108\u0110\u0003@ \u0000\u0109\u0110\u00036\u001b\u0000"+ + "\u010a\u0110\u0003\u0016\u000b\u0000\u010b\u010c\u00050\u0000\u0000\u010c"+ + "\u010d\u0003\n\u0005\u0000\u010d\u010e\u00057\u0000\u0000\u010e\u0110"+ + "\u0001\u0000\u0000\u0000\u010f\u0107\u0001\u0000\u0000\u0000\u010f\u0109"+ + "\u0001\u0000\u0000\u0000\u010f\u010a\u0001\u0000\u0000\u0000\u010f\u010b"+ + "\u0001\u0000\u0000\u0000\u0110\u0116\u0001\u0000\u0000\u0000\u0111\u0112"+ + "\n\u0001\u0000\u0000\u0112\u0113\u0005%\u0000\u0000\u0113\u0115\u0003"+ + "\u001a\r\u0000\u0114\u0111\u0001\u0000\u0000\u0000\u0115\u0118\u0001\u0000"+ + "\u0000\u0000\u0116\u0114\u0001\u0000\u0000\u0000\u0116\u0117\u0001\u0000"+ + "\u0000\u0000\u0117\u0015\u0001\u0000\u0000\u0000\u0118\u0116\u0001\u0000"+ + "\u0000\u0000\u0119\u011a\u0003\u0018\f\u0000\u011a\u0124\u00050\u0000"+ + "\u0000\u011b\u0125\u0005B\u0000\u0000\u011c\u0121\u0003\n\u0005\u0000"+ + "\u011d\u011e\u0005\'\u0000\u0000\u011e\u0120\u0003\n\u0005\u0000\u011f"+ + "\u011d\u0001\u0000\u0000\u0000\u0120\u0123\u0001\u0000\u0000\u0000\u0121"+ + "\u011f\u0001\u0000\u0000\u0000\u0121\u0122\u0001\u0000\u0000\u0000\u0122"+ + "\u0125\u0001\u0000\u0000\u0000\u0123\u0121\u0001\u0000\u0000\u0000\u0124"+ + "\u011b\u0001\u0000\u0000\u0000\u0124\u011c\u0001\u0000\u0000\u0000\u0124"+ + "\u0125\u0001\u0000\u0000\u0000\u0125\u0126\u0001\u0000\u0000\u0000\u0126"+ + "\u0127\u00057\u0000\u0000\u0127\u0017\u0001\u0000\u0000\u0000\u0128\u0129"+ + "\u0003D\"\u0000\u0129\u0019\u0001\u0000\u0000\u0000\u012a\u012b\u0003"+ + "<\u001e\u0000\u012b\u001b\u0001\u0000\u0000\u0000\u012c\u012d\u0005\f"+ + "\u0000\u0000\u012d\u012e\u0003\u001e\u000f\u0000\u012e\u001d\u0001\u0000"+ + "\u0000\u0000\u012f\u0134\u0003 \u0010\u0000\u0130\u0131\u0005\'\u0000"+ + "\u0000\u0131\u0133\u0003 \u0010\u0000\u0132\u0130\u0001\u0000\u0000\u0000"+ + "\u0133\u0136\u0001\u0000\u0000\u0000\u0134\u0132\u0001\u0000\u0000\u0000"+ + "\u0134\u0135\u0001\u0000\u0000\u0000\u0135\u001f\u0001\u0000\u0000\u0000"+ + "\u0136\u0134\u0001\u0000\u0000\u0000\u0137\u0138\u00036\u001b\u0000\u0138"+ + "\u0139\u0005$\u0000\u0000\u0139\u013b\u0001\u0000\u0000\u0000\u013a\u0137"+ + "\u0001\u0000\u0000\u0000\u013a\u013b\u0001\u0000\u0000\u0000\u013b\u013c"+ + "\u0001\u0000\u0000\u0000\u013c\u013d\u0003\n\u0005\u0000\u013d!\u0001"+ + "\u0000\u0000\u0000\u013e\u013f\u0005\u0006\u0000\u0000\u013f\u0144\u0003"+ + "$\u0012\u0000\u0140\u0141\u0005\'\u0000\u0000\u0141\u0143\u0003$\u0012"+ + "\u0000\u0142\u0140\u0001\u0000\u0000\u0000\u0143\u0146\u0001\u0000\u0000"+ + "\u0000\u0144\u0142\u0001\u0000\u0000\u0000\u0144\u0145\u0001\u0000\u0000"+ + "\u0000\u0145\u0148\u0001\u0000\u0000\u0000\u0146\u0144\u0001\u0000\u0000"+ + "\u0000\u0147\u0149\u0003*\u0015\u0000\u0148\u0147\u0001\u0000\u0000\u0000"+ + "\u0148\u0149\u0001\u0000\u0000\u0000\u0149#\u0001\u0000\u0000\u0000\u014a"+ + "\u014b\u0003&\u0013\u0000\u014b\u014c\u0005&\u0000\u0000\u014c\u014e\u0001"+ + "\u0000\u0000\u0000\u014d\u014a\u0001\u0000\u0000\u0000\u014d\u014e\u0001"+ + "\u0000\u0000\u0000\u014e\u014f\u0001\u0000\u0000\u0000\u014f\u0150\u0003"+ + "(\u0014\u0000\u0150%\u0001\u0000\u0000\u0000\u0151\u0152\u0005Q\u0000"+ + "\u0000\u0152\'\u0001\u0000\u0000\u0000\u0153\u0154\u0007\u0002\u0000\u0000"+ + "\u0154)\u0001\u0000\u0000\u0000\u0155\u0156\u0005P\u0000\u0000\u0156\u015b"+ + "\u0005Q\u0000\u0000\u0157\u0158\u0005\'\u0000\u0000\u0158\u015a\u0005"+ + "Q\u0000\u0000\u0159\u0157\u0001\u0000\u0000\u0000\u015a\u015d\u0001\u0000"+ + "\u0000\u0000\u015b\u0159\u0001\u0000\u0000\u0000\u015b\u015c\u0001\u0000"+ + "\u0000\u0000\u015c+\u0001\u0000\u0000\u0000\u015d\u015b\u0001\u0000\u0000"+ + "\u0000\u015e\u015f\u0005\u0013\u0000\u0000\u015f\u0164\u0003$\u0012\u0000"+ + "\u0160\u0161\u0005\'\u0000\u0000\u0161\u0163\u0003$\u0012\u0000\u0162"+ + "\u0160\u0001\u0000\u0000\u0000\u0163\u0166\u0001\u0000\u0000\u0000\u0164"+ + "\u0162\u0001\u0000\u0000\u0000\u0164\u0165\u0001\u0000\u0000\u0000\u0165"+ + "\u0168\u0001\u0000\u0000\u0000\u0166\u0164\u0001\u0000\u0000\u0000\u0167"+ + "\u0169\u00032\u0019\u0000\u0168\u0167\u0001\u0000\u0000\u0000\u0168\u0169"+ + "\u0001\u0000\u0000\u0000\u0169\u016c\u0001\u0000\u0000\u0000\u016a\u016b"+ + "\u0005!\u0000\u0000\u016b\u016d\u0003\u001e\u000f\u0000\u016c\u016a\u0001"+ + "\u0000\u0000\u0000\u016c\u016d\u0001\u0000\u0000\u0000\u016d-\u0001\u0000"+ + "\u0000\u0000\u016e\u016f\u0005\u0004\u0000\u0000\u016f\u0170\u0003\u001e"+ + "\u000f\u0000\u0170/\u0001\u0000\u0000\u0000\u0171\u0173\u0005\u000f\u0000"+ + "\u0000\u0172\u0174\u00032\u0019\u0000\u0173\u0172\u0001\u0000\u0000\u0000"+ + "\u0173\u0174\u0001\u0000\u0000\u0000\u0174\u0177\u0001\u0000\u0000\u0000"+ + "\u0175\u0176\u0005!\u0000\u0000\u0176\u0178\u0003\u001e\u000f\u0000\u0177"+ + "\u0175\u0001\u0000\u0000\u0000\u0177\u0178\u0001\u0000\u0000\u0000\u0178"+ + "1\u0001\u0000\u0000\u0000\u0179\u017e\u00034\u001a\u0000\u017a\u017b\u0005"+ + "\'\u0000\u0000\u017b\u017d\u00034\u001a\u0000\u017c\u017a\u0001\u0000"+ + "\u0000\u0000\u017d\u0180\u0001\u0000\u0000\u0000\u017e\u017c\u0001\u0000"+ + "\u0000\u0000\u017e\u017f\u0001\u0000\u0000\u0000\u017f3\u0001\u0000\u0000"+ + "\u0000\u0180\u017e\u0001\u0000\u0000\u0000\u0181\u0184\u0003 \u0010\u0000"+ + "\u0182\u0183\u0005\u0010\u0000\u0000\u0183\u0185\u0003\n\u0005\u0000\u0184"+ + "\u0182\u0001\u0000\u0000\u0000\u0184\u0185\u0001\u0000\u0000\u0000\u0185"+ + "5\u0001\u0000\u0000\u0000\u0186\u018b\u0003D\"\u0000\u0187\u0188\u0005"+ + ")\u0000\u0000\u0188\u018a\u0003D\"\u0000\u0189\u0187\u0001\u0000\u0000"+ + "\u0000\u018a\u018d\u0001\u0000\u0000\u0000\u018b\u0189\u0001\u0000\u0000"+ + "\u0000\u018b\u018c\u0001\u0000\u0000\u0000\u018c7\u0001\u0000\u0000\u0000"+ + "\u018d\u018b\u0001\u0000\u0000\u0000\u018e\u0193\u0003>\u001f\u0000\u018f"+ + "\u0190\u0005)\u0000\u0000\u0190\u0192\u0003>\u001f\u0000\u0191\u018f\u0001"+ + "\u0000\u0000\u0000\u0192\u0195\u0001\u0000\u0000\u0000\u0193\u0191\u0001"+ + "\u0000\u0000\u0000\u0193\u0194\u0001\u0000\u0000\u0000\u01949\u0001\u0000"+ + "\u0000\u0000\u0195\u0193\u0001\u0000\u0000\u0000\u0196\u019b\u00038\u001c"+ + "\u0000\u0197\u0198\u0005\'\u0000\u0000\u0198\u019a\u00038\u001c\u0000"+ + "\u0199\u0197\u0001\u0000\u0000\u0000\u019a\u019d\u0001\u0000\u0000\u0000"+ + "\u019b\u0199\u0001\u0000\u0000\u0000\u019b\u019c\u0001\u0000\u0000\u0000"+ + "\u019c;\u0001\u0000\u0000\u0000\u019d\u019b\u0001\u0000\u0000\u0000\u019e"+ + "\u019f\u0007\u0003\u0000\u0000\u019f=\u0001\u0000\u0000\u0000\u01a0\u01a4"+ + "\u0005U\u0000\u0000\u01a1\u01a2\u0004\u001f\n\u0000\u01a2\u01a4\u0003"+ + "B!\u0000\u01a3\u01a0\u0001\u0000\u0000\u0000\u01a3\u01a1\u0001\u0000\u0000"+ + "\u0000\u01a4?\u0001\u0000\u0000\u0000\u01a5\u01d0\u00052\u0000\u0000\u01a6"+ + "\u01a7\u0003d2\u0000\u01a7\u01a8\u0005H\u0000\u0000\u01a8\u01d0\u0001"+ + "\u0000\u0000\u0000\u01a9\u01d0\u0003b1\u0000\u01aa\u01d0\u0003d2\u0000"+ + "\u01ab\u01d0\u0003^/\u0000\u01ac\u01d0\u0003B!\u0000\u01ad\u01d0\u0003"+ + "f3\u0000\u01ae\u01af\u0005F\u0000\u0000\u01af\u01b4\u0003`0\u0000\u01b0"+ + "\u01b1\u0005\'\u0000\u0000\u01b1\u01b3\u0003`0\u0000\u01b2\u01b0\u0001"+ + "\u0000\u0000\u0000\u01b3\u01b6\u0001\u0000\u0000\u0000\u01b4\u01b2\u0001"+ + "\u0000\u0000\u0000\u01b4\u01b5\u0001\u0000\u0000\u0000\u01b5\u01b7\u0001"+ + "\u0000\u0000\u0000\u01b6\u01b4\u0001\u0000\u0000\u0000\u01b7\u01b8\u0005"+ + "G\u0000\u0000\u01b8\u01d0\u0001\u0000\u0000\u0000\u01b9\u01ba\u0005F\u0000"+ + "\u0000\u01ba\u01bf\u0003^/\u0000\u01bb\u01bc\u0005\'\u0000\u0000\u01bc"+ + "\u01be\u0003^/\u0000\u01bd\u01bb\u0001\u0000\u0000\u0000\u01be\u01c1\u0001"+ + "\u0000\u0000\u0000\u01bf\u01bd\u0001\u0000\u0000\u0000\u01bf\u01c0\u0001"+ + "\u0000\u0000\u0000\u01c0\u01c2\u0001\u0000\u0000\u0000\u01c1\u01bf\u0001"+ + "\u0000\u0000\u0000\u01c2\u01c3\u0005G\u0000\u0000\u01c3\u01d0\u0001\u0000"+ + "\u0000\u0000\u01c4\u01c5\u0005F\u0000\u0000\u01c5\u01ca\u0003f3\u0000"+ + "\u01c6\u01c7\u0005\'\u0000\u0000\u01c7\u01c9\u0003f3\u0000\u01c8\u01c6"+ + "\u0001\u0000\u0000\u0000\u01c9\u01cc\u0001\u0000\u0000\u0000\u01ca\u01c8"+ + "\u0001\u0000\u0000\u0000\u01ca\u01cb\u0001\u0000\u0000\u0000\u01cb\u01cd"+ + "\u0001\u0000\u0000\u0000\u01cc\u01ca\u0001\u0000\u0000\u0000\u01cd\u01ce"+ + "\u0005G\u0000\u0000\u01ce\u01d0\u0001\u0000\u0000\u0000\u01cf\u01a5\u0001"+ + "\u0000\u0000\u0000\u01cf\u01a6\u0001\u0000\u0000\u0000\u01cf\u01a9\u0001"+ + "\u0000\u0000\u0000\u01cf\u01aa\u0001\u0000\u0000\u0000\u01cf\u01ab\u0001"+ + "\u0000\u0000\u0000\u01cf\u01ac\u0001\u0000\u0000\u0000\u01cf\u01ad\u0001"+ + "\u0000\u0000\u0000\u01cf\u01ae\u0001\u0000\u0000\u0000\u01cf\u01b9\u0001"+ + "\u0000\u0000\u0000\u01cf\u01c4\u0001\u0000\u0000\u0000\u01d0A\u0001\u0000"+ + "\u0000\u0000\u01d1\u01d4\u00055\u0000\u0000\u01d2\u01d4\u0005E\u0000\u0000"+ + "\u01d3\u01d1\u0001\u0000\u0000\u0000\u01d3\u01d2\u0001\u0000\u0000\u0000"+ + "\u01d4C\u0001\u0000\u0000\u0000\u01d5\u01d9\u0003<\u001e\u0000\u01d6\u01d7"+ + "\u0004\"\u000b\u0000\u01d7\u01d9\u0003B!\u0000\u01d8\u01d5\u0001\u0000"+ + "\u0000\u0000\u01d8\u01d6\u0001\u0000\u0000\u0000\u01d9E\u0001\u0000\u0000"+ + "\u0000\u01da\u01db\u0005\t\u0000\u0000\u01db\u01dc\u0005\u001f\u0000\u0000"+ + "\u01dcG\u0001\u0000\u0000\u0000\u01dd\u01de\u0005\u000e\u0000\u0000\u01de"+ + "\u01e3\u0003J%\u0000\u01df\u01e0\u0005\'\u0000\u0000\u01e0\u01e2\u0003"+ + "J%\u0000\u01e1\u01df\u0001\u0000\u0000\u0000\u01e2\u01e5\u0001\u0000\u0000"+ + "\u0000\u01e3\u01e1\u0001\u0000\u0000\u0000\u01e3\u01e4\u0001\u0000\u0000"+ + "\u0000\u01e4I\u0001\u0000\u0000\u0000\u01e5\u01e3\u0001\u0000\u0000\u0000"+ + "\u01e6\u01e8\u0003\n\u0005\u0000\u01e7\u01e9\u0007\u0004\u0000\u0000\u01e8"+ + "\u01e7\u0001\u0000\u0000\u0000\u01e8\u01e9\u0001\u0000\u0000\u0000\u01e9"+ + "\u01ec\u0001\u0000\u0000\u0000\u01ea\u01eb\u00053\u0000\u0000\u01eb\u01ed"+ + "\u0007\u0005\u0000\u0000\u01ec\u01ea\u0001\u0000\u0000\u0000\u01ec\u01ed"+ + "\u0001\u0000\u0000\u0000\u01edK\u0001\u0000\u0000\u0000\u01ee\u01ef\u0005"+ + "\b\u0000\u0000\u01ef\u01f0\u0003:\u001d\u0000\u01f0M\u0001\u0000\u0000"+ + "\u0000\u01f1\u01f2\u0005\u0002\u0000\u0000\u01f2\u01f3\u0003:\u001d\u0000"+ + "\u01f3O\u0001\u0000\u0000\u0000\u01f4\u01f5\u0005\u000b\u0000\u0000\u01f5"+ + "\u01fa\u0003R)\u0000\u01f6\u01f7\u0005\'\u0000\u0000\u01f7\u01f9\u0003"+ + "R)\u0000\u01f8\u01f6\u0001\u0000\u0000\u0000\u01f9\u01fc\u0001\u0000\u0000"+ + "\u0000\u01fa\u01f8\u0001\u0000\u0000\u0000\u01fa\u01fb\u0001\u0000\u0000"+ + "\u0000\u01fbQ\u0001\u0000\u0000\u0000\u01fc\u01fa\u0001\u0000\u0000\u0000"+ + "\u01fd\u01fe\u00038\u001c\u0000\u01fe\u01ff\u0005Y\u0000\u0000\u01ff\u0200"+ + "\u00038\u001c\u0000\u0200S\u0001\u0000\u0000\u0000\u0201\u0202\u0005\u0001"+ + "\u0000\u0000\u0202\u0203\u0003\u0014\n\u0000\u0203\u0205\u0003f3\u0000"+ + "\u0204\u0206\u0003Z-\u0000\u0205\u0204\u0001\u0000\u0000\u0000\u0205\u0206"+ + "\u0001\u0000\u0000\u0000\u0206U\u0001\u0000\u0000\u0000\u0207\u0208\u0005"+ + "\u0007\u0000\u0000\u0208\u0209\u0003\u0014\n\u0000\u0209\u020a\u0003f"+ + "3\u0000\u020aW\u0001\u0000\u0000\u0000\u020b\u020c\u0005\n\u0000\u0000"+ + "\u020c\u020d\u00036\u001b\u0000\u020dY\u0001\u0000\u0000\u0000\u020e\u0213"+ + "\u0003\\.\u0000\u020f\u0210\u0005\'\u0000\u0000\u0210\u0212\u0003\\.\u0000"+ + "\u0211\u020f\u0001\u0000\u0000\u0000\u0212\u0215\u0001\u0000\u0000\u0000"+ + "\u0213\u0211\u0001\u0000\u0000\u0000\u0213\u0214\u0001\u0000\u0000\u0000"+ + "\u0214[\u0001\u0000\u0000\u0000\u0215\u0213\u0001\u0000\u0000\u0000\u0216"+ + "\u0217\u0003<\u001e\u0000\u0217\u0218\u0005$\u0000\u0000\u0218\u0219\u0003"+ + "@ \u0000\u0219]\u0001\u0000\u0000\u0000\u021a\u021b\u0007\u0006\u0000"+ + "\u0000\u021b_\u0001\u0000\u0000\u0000\u021c\u021f\u0003b1\u0000\u021d"+ + "\u021f\u0003d2\u0000\u021e\u021c\u0001\u0000\u0000\u0000\u021e\u021d\u0001"+ + "\u0000\u0000\u0000\u021fa\u0001\u0000\u0000\u0000\u0220\u0222\u0007\u0000"+ + "\u0000\u0000\u0221\u0220\u0001\u0000\u0000\u0000\u0221\u0222\u0001\u0000"+ + "\u0000\u0000\u0222\u0223\u0001\u0000\u0000\u0000\u0223\u0224\u0005 \u0000"+ + "\u0000\u0224c\u0001\u0000\u0000\u0000\u0225\u0227\u0007\u0000\u0000\u0000"+ + "\u0226\u0225\u0001\u0000\u0000\u0000\u0226\u0227\u0001\u0000\u0000\u0000"+ + "\u0227\u0228\u0001\u0000\u0000\u0000\u0228\u0229\u0005\u001f\u0000\u0000"+ + "\u0229e\u0001\u0000\u0000\u0000\u022a\u022b\u0005\u001e\u0000\u0000\u022b"+ + "g\u0001\u0000\u0000\u0000\u022c\u022d\u0007\u0007\u0000\u0000\u022di\u0001"+ + "\u0000\u0000\u0000\u022e\u022f\u0005\u0005\u0000\u0000\u022f\u0230\u0003"+ + "l6\u0000\u0230k\u0001\u0000\u0000\u0000\u0231\u0232\u0005F\u0000\u0000"+ + "\u0232\u0233\u0003\u0002\u0001\u0000\u0233\u0234\u0005G\u0000\u0000\u0234"+ + "m\u0001\u0000\u0000\u0000\u0235\u0236\u0005\r\u0000\u0000\u0236\u0237"+ + "\u0005i\u0000\u0000\u0237o\u0001\u0000\u0000\u0000\u0238\u0239\u0005\u0003"+ + "\u0000\u0000\u0239\u023c\u0005_\u0000\u0000\u023a\u023b\u0005]\u0000\u0000"+ + "\u023b\u023d\u00038\u001c\u0000\u023c\u023a\u0001\u0000\u0000\u0000\u023c"+ + "\u023d\u0001\u0000\u0000\u0000\u023d\u0247\u0001\u0000\u0000\u0000\u023e"+ + "\u023f\u0005^\u0000\u0000\u023f\u0244\u0003r9\u0000\u0240\u0241\u0005"+ + "\'\u0000\u0000\u0241\u0243\u0003r9\u0000\u0242\u0240\u0001\u0000\u0000"+ + "\u0000\u0243\u0246\u0001\u0000\u0000\u0000\u0244\u0242\u0001\u0000\u0000"+ + "\u0000\u0244\u0245\u0001\u0000\u0000\u0000\u0245\u0248\u0001\u0000\u0000"+ + "\u0000\u0246\u0244\u0001\u0000\u0000\u0000\u0247\u023e\u0001\u0000\u0000"+ + "\u0000\u0247\u0248\u0001\u0000\u0000\u0000\u0248q\u0001\u0000\u0000\u0000"+ + "\u0249\u024a\u00038\u001c\u0000\u024a\u024b\u0005$\u0000\u0000\u024b\u024d"+ + "\u0001\u0000\u0000\u0000\u024c\u0249\u0001\u0000\u0000\u0000\u024c\u024d"+ + "\u0001\u0000\u0000\u0000\u024d\u024e\u0001\u0000\u0000\u0000\u024e\u024f"+ + "\u00038\u001c\u0000\u024fs\u0001\u0000\u0000\u0000\u0250\u0251\u0005\u0012"+ + "\u0000\u0000\u0251\u0252\u0003$\u0012\u0000\u0252\u0253\u0005]\u0000\u0000"+ + "\u0253\u0254\u0003:\u001d\u0000\u0254u\u0001\u0000\u0000\u0000\u0255\u0256"+ + "\u0005\u0011\u0000\u0000\u0256\u0259\u00032\u0019\u0000\u0257\u0258\u0005"+ + "!\u0000\u0000\u0258\u025a\u0003\u001e\u000f\u0000\u0259\u0257\u0001\u0000"+ + "\u0000\u0000\u0259\u025a\u0001\u0000\u0000\u0000\u025aw\u0001\u0000\u0000"+ + "\u0000\u025b\u025d\u0007\b\u0000\u0000\u025c\u025b\u0001\u0000\u0000\u0000"+ + "\u025c\u025d\u0001\u0000\u0000\u0000\u025d\u025e\u0001\u0000\u0000\u0000"+ + "\u025e\u025f\u0005\u0014\u0000\u0000\u025f\u0260\u0003z=\u0000\u0260\u0261"+ + "\u0003|>\u0000\u0261y\u0001\u0000\u0000\u0000\u0262\u0265\u0003<\u001e"+ + "\u0000\u0263\u0264\u0005Y\u0000\u0000\u0264\u0266\u0003<\u001e\u0000\u0265"+ + "\u0263\u0001\u0000\u0000\u0000\u0265\u0266\u0001\u0000\u0000\u0000\u0266"+ + "{\u0001\u0000\u0000\u0000\u0267\u0268\u0005]\u0000\u0000\u0268\u026d\u0003"+ + "~?\u0000\u0269\u026a\u0005\'\u0000\u0000\u026a\u026c\u0003~?\u0000\u026b"+ + "\u0269\u0001\u0000\u0000\u0000\u026c\u026f\u0001\u0000\u0000\u0000\u026d"+ + "\u026b\u0001\u0000\u0000\u0000\u026d\u026e\u0001\u0000\u0000\u0000\u026e"+ + "}\u0001\u0000\u0000\u0000\u026f\u026d\u0001\u0000\u0000\u0000\u0270\u0271"+ + "\u0003\u0010\b\u0000\u0271\u007f\u0001\u0000\u0000\u0000=\u008b\u0094"+ + "\u00a8\u00b4\u00bd\u00c5\u00ca\u00d2\u00d4\u00d9\u00e0\u00e5\u00ea\u00f4"+ + "\u00fa\u0102\u0104\u010f\u0116\u0121\u0124\u0134\u013a\u0144\u0148\u014d"+ + "\u015b\u0164\u0168\u016c\u0173\u0177\u017e\u0184\u018b\u0193\u019b\u01a3"+ + "\u01b4\u01bf\u01ca\u01cf\u01d3\u01d8\u01e3\u01e8\u01ec\u01fa\u0205\u0213"+ + "\u021e\u0221\u0226\u023c\u0244\u0247\u024c\u0259\u025c\u0265\u026d"; public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray()); static { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java index 6071219839bab..2ee0efe52dfbe 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java @@ -452,30 +452,6 @@ public class EsqlBaseParserBaseListener implements EsqlBaseParserListener { *

The default implementation does nothing.

*/ @Override public void exitMetadata(EsqlBaseParser.MetadataContext ctx) { } - /** - * {@inheritDoc} - * - *

The default implementation does nothing.

- */ - @Override public void enterMetadataOption(EsqlBaseParser.MetadataOptionContext ctx) { } - /** - * {@inheritDoc} - * - *

The default implementation does nothing.

- */ - @Override public void exitMetadataOption(EsqlBaseParser.MetadataOptionContext ctx) { } - /** - * {@inheritDoc} - * - *

The default implementation does nothing.

- */ - @Override public void enterDeprecated_metadata(EsqlBaseParser.Deprecated_metadataContext ctx) { } - /** - * {@inheritDoc} - * - *

The default implementation does nothing.

- */ - @Override public void exitDeprecated_metadata(EsqlBaseParser.Deprecated_metadataContext ctx) { } /** * {@inheritDoc} * diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java index afe7146923791..c5c1b0dfa7d7d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java @@ -272,20 +272,6 @@ public class EsqlBaseParserBaseVisitor extends AbstractParseTreeVisitor im * {@link #visitChildren} on {@code ctx}.

*/ @Override public T visitMetadata(EsqlBaseParser.MetadataContext ctx) { return visitChildren(ctx); } - /** - * {@inheritDoc} - * - *

The default implementation returns the result of calling - * {@link #visitChildren} on {@code ctx}.

- */ - @Override public T visitMetadataOption(EsqlBaseParser.MetadataOptionContext ctx) { return visitChildren(ctx); } - /** - * {@inheritDoc} - * - *

The default implementation returns the result of calling - * {@link #visitChildren} on {@code ctx}.

- */ - @Override public T visitDeprecated_metadata(EsqlBaseParser.Deprecated_metadataContext ctx) { return visitChildren(ctx); } /** * {@inheritDoc} * diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java index 0faca2541c9ad..f45184e920658 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java @@ -415,26 +415,6 @@ public interface EsqlBaseParserListener extends ParseTreeListener { * @param ctx the parse tree */ void exitMetadata(EsqlBaseParser.MetadataContext ctx); - /** - * Enter a parse tree produced by {@link EsqlBaseParser#metadataOption}. - * @param ctx the parse tree - */ - void enterMetadataOption(EsqlBaseParser.MetadataOptionContext ctx); - /** - * Exit a parse tree produced by {@link EsqlBaseParser#metadataOption}. - * @param ctx the parse tree - */ - void exitMetadataOption(EsqlBaseParser.MetadataOptionContext ctx); - /** - * Enter a parse tree produced by {@link EsqlBaseParser#deprecated_metadata}. - * @param ctx the parse tree - */ - void enterDeprecated_metadata(EsqlBaseParser.Deprecated_metadataContext ctx); - /** - * Exit a parse tree produced by {@link EsqlBaseParser#deprecated_metadata}. - * @param ctx the parse tree - */ - void exitDeprecated_metadata(EsqlBaseParser.Deprecated_metadataContext ctx); /** * Enter a parse tree produced by {@link EsqlBaseParser#metricsCommand}. * @param ctx the parse tree diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java index e91cd6670e971..30c5e0ce78092 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java @@ -254,18 +254,6 @@ public interface EsqlBaseParserVisitor extends ParseTreeVisitor { * @return the visitor result */ T visitMetadata(EsqlBaseParser.MetadataContext ctx); - /** - * Visit a parse tree produced by {@link EsqlBaseParser#metadataOption}. - * @param ctx the parse tree - * @return the visitor result - */ - T visitMetadataOption(EsqlBaseParser.MetadataOptionContext ctx); - /** - * Visit a parse tree produced by {@link EsqlBaseParser#deprecated_metadata}. - * @param ctx the parse tree - * @return the visitor result - */ - T visitDeprecated_metadata(EsqlBaseParser.Deprecated_metadataContext ctx); /** * Visit a parse tree produced by {@link EsqlBaseParser#metricsCommand}. * @param ctx the parse tree diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java index 4edd0470058db..4b7c0118acda3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java @@ -35,7 +35,6 @@ import org.elasticsearch.xpack.esql.expression.Order; import org.elasticsearch.xpack.esql.expression.UnresolvedNamePattern; import org.elasticsearch.xpack.esql.expression.function.UnresolvedFunction; -import org.elasticsearch.xpack.esql.parser.EsqlBaseParser.MetadataOptionContext; import org.elasticsearch.xpack.esql.plan.TableIdentifier; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; import org.elasticsearch.xpack.esql.plan.logical.Dissect; @@ -72,7 +71,6 @@ import java.util.function.Function; import static java.util.Collections.emptyList; -import static org.elasticsearch.common.logging.HeaderWarning.addWarning; import static org.elasticsearch.xpack.esql.core.util.StringUtils.WILDCARD; import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputExpressions; import static org.elasticsearch.xpack.esql.parser.ParserUtils.source; @@ -261,21 +259,7 @@ public LogicalPlan visitFromCommand(EsqlBaseParser.FromCommandContext ctx) { TableIdentifier table = new TableIdentifier(source, null, visitIndexPattern(ctx.indexPattern())); Map metadataMap = new LinkedHashMap<>(); if (ctx.metadata() != null) { - var deprecatedContext = ctx.metadata().deprecated_metadata(); - MetadataOptionContext metadataOptionContext = null; - if (deprecatedContext != null) { - var s = source(deprecatedContext).source(); - addWarning( - "Line {}:{}: Square brackets '[]' need to be removed in FROM METADATA declaration", - s.getLineNumber(), - s.getColumnNumber() - ); - metadataOptionContext = deprecatedContext.metadataOption(); - } else { - metadataOptionContext = ctx.metadata().metadataOption(); - - } - for (var c : metadataOptionContext.UNQUOTED_SOURCE()) { + for (var c : ctx.metadata().UNQUOTED_SOURCE()) { String id = c.getText(); Source src = source(c); if (MetadataAttribute.isSupported(id) == false // TODO: drop check below once METADATA_SCORE is no longer snapshot-only diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java index a4712ae77b5d8..a6243d25ba579 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java @@ -625,11 +625,8 @@ public void testInvalidQuotingAsFromIndexPattern() { expectError("FROM \"foo\"bar\"", ": token recognition error at: '\"'"); expectError("FROM \"foo\"\"bar\"", ": extraneous input '\"bar\"' expecting "); - expectError("FROM \"\"\"foo\"\"\"bar\"\"\"", ": mismatched input 'bar' expecting {, '|', ',', OPENING_BRACKET, 'metadata'}"); - expectError( - "FROM \"\"\"foo\"\"\"\"\"\"bar\"\"\"", - ": mismatched input '\"bar\"' expecting {, '|', ',', OPENING_BRACKET, 'metadata'}" - ); + expectError("FROM \"\"\"foo\"\"\"bar\"\"\"", ": mismatched input 'bar' expecting {, '|', ',', 'metadata'}"); + expectError("FROM \"\"\"foo\"\"\"\"\"\"bar\"\"\"", ": mismatched input '\"bar\"' expecting {, '|', ',', 'metadata'}"); } public void testInvalidQuotingAsMetricsIndexPattern() { @@ -909,10 +906,7 @@ public void testDeprecatedIsNullFunction() { public void testMetadataFieldOnOtherSources() { expectError("row a = 1 metadata _index", "line 1:20: extraneous input '_index' expecting "); expectError("show info metadata _index", "line 1:11: token recognition error at: 'm'"); - expectError( - "explain [from foo] metadata _index", - "line 1:20: mismatched input 'metadata' expecting {'|', ',', OPENING_BRACKET, ']', 'metadata'}" - ); + expectError("explain [from foo] metadata _index", "line 1:20: mismatched input 'metadata' expecting {'|', ',', ']', 'metadata'}"); } public void testMetadataFieldMultipleDeclarations() { @@ -2357,4 +2351,11 @@ public void testMatchOperatorFieldCasting() { assertThat(matchField.name(), equalTo("field")); assertThat(match.query().fold(FoldContext.small()), equalTo("value")); } + + public void testFailingMetadataWithSquareBrackets() { + expectError( + "FROM test [METADATA _index] | STATS count(*)", + "line 1:11: mismatched input '[' expecting {, '|', ',', 'metadata'}" + ); + } } diff --git a/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/src/test/resources/rest-api-spec/test/querying_cluster/80_esql.yml b/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/src/test/resources/rest-api-spec/test/querying_cluster/80_esql.yml index 1b435c551fbe9..41bc0a96e737e 100644 --- a/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/src/test/resources/rest-api-spec/test/querying_cluster/80_esql.yml +++ b/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/src/test/resources/rest-api-spec/test/querying_cluster/80_esql.yml @@ -92,8 +92,6 @@ teardown: - do: catch: bad_request - allowed_warnings: - - "Line 1:21: Square brackets '[]' need to be removed in FROM METADATA declaration" headers: { Authorization: "Basic am9lOnMza3JpdC1wYXNzd29yZA==" } esql.query: body: @@ -104,12 +102,10 @@ teardown: - do: catch: bad_request - allowed_warnings: - - "Line 1:21: Square brackets '[]' need to be removed in FROM METADATA declaration" headers: { Authorization: "Basic am9lOnMza3JpdC1wYXNzd29yZA==" } esql.query: body: - query: 'FROM *:esql*,esql_* [METADATA _index] | sort cost | KEEP _index, tag, cost | LIMIT 10' + query: 'FROM *:esql*,esql_* METADATA _index | sort cost | KEEP _index, tag, cost | LIMIT 10' filter: range: since: From 35341330b2942a6a05af57567a1228223132526f Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Tue, 14 Jan 2025 22:44:04 +1100 Subject: [PATCH 17/31] Mute org.elasticsearch.xpack.searchablesnapshots.minio.MinioSearchableSnapshotsIT org.elasticsearch.xpack.searchablesnapshots.minio.MinioSearchableSnapshotsIT #120101 --- muted-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 3e69b7be0b2c3..0eb80b79b74c1 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -267,6 +267,8 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/118406 - class: org.elasticsearch.xpack.ml.integration.DatafeedJobsIT issue: https://github.com/elastic/elasticsearch/issues/120088 +- class: org.elasticsearch.xpack.searchablesnapshots.minio.MinioSearchableSnapshotsIT + issue: https://github.com/elastic/elasticsearch/issues/120101 # Examples: # From 2fe55dbc3ec83cc280ac2f791bfca1d9a8ee7d6d Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 14 Jan 2025 12:57:04 +0100 Subject: [PATCH 18/31] Minimize granted and denied fields in FLS (#119453) Instead of only determinizing, we can minimize the intermediate automata during FLS automaton construction. This brings the codebase back to its original form, before the Lucene 10 upgrade. Follow up to: https://github.com/elastic/elasticsearch/pull/119309 --- .../core/security/authz/permission/FieldPermissions.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java index 235d7419d2bf0..c46f1350776b1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java @@ -18,6 +18,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.lucene.util.automaton.MinimizationOperations; import org.elasticsearch.plugins.FieldPredicate; import org.elasticsearch.xpack.core.security.authz.accesscontrol.FieldSubsetReader; import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsDefinition.FieldGrantExcludeGroup; @@ -172,12 +173,8 @@ public static Automaton buildPermittedFieldsAutomaton(final String[] grantedFiel deniedFieldsAutomaton = Automatons.patterns(deniedFields); } - grantedFieldsAutomaton = Operations.removeDeadStates( - Operations.determinize(grantedFieldsAutomaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT) - ); - deniedFieldsAutomaton = Operations.removeDeadStates( - Operations.determinize(deniedFieldsAutomaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT) - ); + grantedFieldsAutomaton = MinimizationOperations.minimize(grantedFieldsAutomaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT); + deniedFieldsAutomaton = MinimizationOperations.minimize(deniedFieldsAutomaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT); if (Automatons.subsetOf(deniedFieldsAutomaton, grantedFieldsAutomaton) == false) { throw new ElasticsearchSecurityException( From a7d831e3009cc2fc2170e73861577e5fd18a4dd7 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 15 Jan 2025 00:45:54 +1100 Subject: [PATCH 19/31] Mute org.elasticsearch.repositories.s3.S3RepositoryThirdPartyTests org.elasticsearch.repositories.s3.S3RepositoryThirdPartyTests #120115 --- muted-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 0eb80b79b74c1..96f215b8b6815 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -269,6 +269,8 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/120088 - class: org.elasticsearch.xpack.searchablesnapshots.minio.MinioSearchableSnapshotsIT issue: https://github.com/elastic/elasticsearch/issues/120101 +- class: org.elasticsearch.repositories.s3.S3RepositoryThirdPartyTests + issue: https://github.com/elastic/elasticsearch/issues/120115 # Examples: # From c638f746763733532e3c9db018ea72916955ac3a Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 15 Jan 2025 00:49:53 +1100 Subject: [PATCH 20/31] Mute org.elasticsearch.repositories.s3.RepositoryS3MinioBasicCredentialsRestIT org.elasticsearch.repositories.s3.RepositoryS3MinioBasicCredentialsRestIT #120117 --- muted-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 96f215b8b6815..9cf2ef065f82f 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -271,6 +271,8 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/120101 - class: org.elasticsearch.repositories.s3.S3RepositoryThirdPartyTests issue: https://github.com/elastic/elasticsearch/issues/120115 +- class: org.elasticsearch.repositories.s3.RepositoryS3MinioBasicCredentialsRestIT + issue: https://github.com/elastic/elasticsearch/issues/120117 # Examples: # From fc8e2fcc22d354ebfd37d99aa9f024e6a9989dda Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Tue, 14 Jan 2025 15:07:48 +0100 Subject: [PATCH 21/31] Move source mode setting from SourceFieldMapper to IndexSettings (#120096) Here we move the `index.mapping.source.mode` setting to `IndexSettings` because of dependencies and because of the initialisation order of static fields for classes `IndexSettings` and `SourceFieldMapper`. Not initialising settings `index.mode`, `index.mapping.source.mode`, and `index.recovery.use_synthetic_source` in the right order results in multiple `NullPointerException`. This work is done to simplify another PR #119110 --- .../admin/cluster/stats/MappingStats.java | 4 +- .../metadata/MetadataCreateIndexService.java | 3 +- .../common/settings/IndexScopedSettings.java | 3 +- .../elasticsearch/index/IndexSettings.java | 97 +++++++++++-------- .../index/mapper/SourceFieldMapper.java | 16 +-- .../snapshots/RestoreService.java | 3 +- .../cluster/stats/MappingStatsTests.java | 8 +- ...neSyntheticSourceChangesSnapshotTests.java | 4 +- .../TranslogOperationAsserterTests.java | 8 +- ...edSourceFieldMapperConfigurationTests.java | 3 +- .../index/mapper/SourceFieldMapperTests.java | 30 +++--- .../index/mapper/MapperTestCase.java | 2 +- .../xpack/ccr/FollowIndexIT.java | 2 +- .../TransportResumeFollowActionTests.java | 3 +- .../SourceOnlySnapshotShardTests.java | 4 +- .../logsdb/LogsDBUsageTransportAction.java | 5 +- .../LogsdbIndexModeSettingsProvider.java | 6 +- .../logsdb/LegacyLicenceIntegrationTests.java | 6 +- .../LogsdbIndexModeSettingsProviderTests.java | 10 +- ...dexSettingsProviderLegacyLicenseTests.java | 16 +-- 20 files changed, 115 insertions(+), 118 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java index 1bc2e1d13c864..29a124b3d0b20 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java @@ -21,7 +21,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.Nullable; import org.elasticsearch.features.NodeFeature; -import org.elasticsearch.index.mapper.SourceFieldMapper; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.xcontent.ToXContentFragment; import org.elasticsearch.xcontent.XContentBuilder; @@ -71,7 +71,7 @@ public static MappingStats of(Metadata metadata, Runnable ensureNotCancelled) { } AnalysisStats.countMapping(mappingCounts, indexMetadata); - var sourceMode = SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexMetadata.getSettings()); + var sourceMode = IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexMetadata.getSettings()); sourceModeUsageCount.merge(sourceMode.toString().toLowerCase(Locale.ENGLISH), 1, Integer::sum); } final AtomicLong totalFieldCount = new AtomicLong(); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index fb19a3f04fcec..e24d921ba78fd 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -69,7 +69,6 @@ import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperService.MergeReason; -import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.indices.IndexCreationException; import org.elasticsearch.indices.IndicesService; @@ -1586,7 +1585,7 @@ static void validateCloneIndex( private static final Set UNMODIFIABLE_SETTINGS_DURING_RESIZE = Set.of( IndexSettings.MODE.getKey(), - SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), + IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey(), diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index afee2491672ef..3c1f53ca4a2c9 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -36,7 +36,6 @@ import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.store.FsDirectoryFactory; import org.elasticsearch.index.store.Store; @@ -191,7 +190,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { FieldMapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING, IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING, IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING, - SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING, + IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING, IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING, InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT, diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 68f334b10ea52..766b6ecbc7b9d 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -52,7 +52,6 @@ import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING; import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING; import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING; -import static org.elasticsearch.index.mapper.SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING; /** * This class encapsulates all index level settings and handles settings updates. @@ -655,48 +654,6 @@ public Iterator> settings() { Property.Final ); - public static final Setting RECOVERY_USE_SYNTHETIC_SOURCE_SETTING = Setting.boolSetting( - "index.recovery.use_synthetic_source", - false, - new Setting.Validator<>() { - @Override - public void validate(Boolean value) {} - - @Override - public void validate(Boolean enabled, Map, Object> settings) { - if (enabled == false) { - return; - } - - // Verify if synthetic source is enabled on the index; fail if it is not - var indexMode = (IndexMode) settings.get(MODE); - if (indexMode.defaultSourceMode() != SourceFieldMapper.Mode.SYNTHETIC) { - var sourceMode = (SourceFieldMapper.Mode) settings.get(INDEX_MAPPER_SOURCE_MODE_SETTING); - if (sourceMode != SourceFieldMapper.Mode.SYNTHETIC) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "The setting [%s] is only permitted when [%s] is set to [%s]. Current mode: [%s].", - RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), - INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), - SourceFieldMapper.Mode.SYNTHETIC.name(), - sourceMode.name() - ) - ); - } - } - } - - @Override - public Iterator> settings() { - List> res = List.of(INDEX_MAPPER_SOURCE_MODE_SETTING, MODE); - return res.iterator(); - } - }, - Property.IndexScope, - Property.Final - ); - /** * Returns true if TSDB encoding is enabled. The default is true */ @@ -753,6 +710,60 @@ public Iterator> settings() { Property.ServerlessPublic ); + public static final Setting INDEX_MAPPER_SOURCE_MODE_SETTING = Setting.enumSetting( + SourceFieldMapper.Mode.class, + settings -> { + final IndexMode indexMode = IndexSettings.MODE.get(settings); + return indexMode.defaultSourceMode().name(); + }, + "index.mapping.source.mode", + value -> {}, + Setting.Property.Final, + Setting.Property.IndexScope + ); + + public static final Setting RECOVERY_USE_SYNTHETIC_SOURCE_SETTING = Setting.boolSetting( + "index.recovery.use_synthetic_source", + false, + new Setting.Validator<>() { + @Override + public void validate(Boolean value) {} + + @Override + public void validate(Boolean enabled, Map, Object> settings) { + if (enabled == false) { + return; + } + + // Verify if synthetic source is enabled on the index; fail if it is not + var indexMode = (IndexMode) settings.get(MODE); + if (indexMode.defaultSourceMode() != SourceFieldMapper.Mode.SYNTHETIC) { + var sourceMode = (SourceFieldMapper.Mode) settings.get(INDEX_MAPPER_SOURCE_MODE_SETTING); + if (sourceMode != SourceFieldMapper.Mode.SYNTHETIC) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "The setting [%s] is only permitted when [%s] is set to [%s]. Current mode: [%s].", + RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), + INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), + SourceFieldMapper.Mode.SYNTHETIC.name(), + sourceMode.name() + ) + ); + } + } + } + + @Override + public Iterator> settings() { + List> res = List.of(INDEX_MAPPER_SOURCE_MODE_SETTING, MODE); + return res.iterator(); + } + }, + Property.IndexScope, + Property.Final + ); + /** * Legacy index setting, kept for 7.x BWC compatibility. This setting has no effect in 8.x. Do not use. * TODO: Remove in 9.0 diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index 29acdf200692d..36335be58ce94 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -18,7 +18,6 @@ import org.elasticsearch.common.Explicit; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.core.Nullable; @@ -70,11 +69,6 @@ public class SourceFieldMapper extends MetadataFieldMapper { public static final String LOSSY_PARAMETERS_ALLOWED_SETTING_NAME = "index.lossy.source-mapping-parameters"; - public static final Setting INDEX_MAPPER_SOURCE_MODE_SETTING = Setting.enumSetting(SourceFieldMapper.Mode.class, settings -> { - final IndexMode indexMode = IndexSettings.MODE.get(settings); - return indexMode.defaultSourceMode().name(); - }, "index.mapping.source.mode", value -> {}, Setting.Property.Final, Setting.Property.IndexScope); - public static final String DEPRECATION_WARNING = "Configuring source mode in mappings is deprecated and will be removed " + "in future versions. Use [index.mapping.source.mode] index setting instead."; @@ -264,8 +258,8 @@ public SourceFieldMapper build() { private Mode resolveSourceMode() { // If the `index.mapping.source.mode` exists it takes precedence to determine the source mode for `_source` // otherwise the mode is determined according to `_source.mode`. - if (INDEX_MAPPER_SOURCE_MODE_SETTING.exists(settings)) { - return INDEX_MAPPER_SOURCE_MODE_SETTING.get(settings); + if (IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.exists(settings)) { + return IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(settings); } // If `_source.mode` is not set we need to apply a default according to index mode. @@ -297,7 +291,7 @@ private static SourceFieldMapper resolveStaticInstance(final Mode sourceMode) { return DEFAULT; } - final Mode settingSourceMode = INDEX_MAPPER_SOURCE_MODE_SETTING.get(c.getSettings()); + final Mode settingSourceMode = IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(c.getSettings()); // Needed for bwc so that "mode" is not serialized in case of standard index with stored source. if (indexMode == IndexMode.STANDARD && settingSourceMode == Mode.STORED) { return DEFAULT; @@ -482,11 +476,11 @@ public boolean isSynthetic() { } public static boolean isSynthetic(IndexSettings indexSettings) { - return INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexSettings.getSettings()) == SourceFieldMapper.Mode.SYNTHETIC; + return IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexSettings.getSettings()) == SourceFieldMapper.Mode.SYNTHETIC; } public static boolean isStored(IndexSettings indexSettings) { - return INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexSettings.getSettings()) == Mode.STORED; + return IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexSettings.getSettings()) == Mode.STORED; } public boolean isDisabled() { diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index f260c7aad30e5..debe3d6e6bd92 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -70,7 +70,6 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.Mapping; -import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.shard.IndexLongFieldRange; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; @@ -158,7 +157,7 @@ public final class RestoreService implements ClusterStateApplier { SETTING_CREATION_DATE, SETTING_HISTORY_UUID, IndexSettings.MODE.getKey(), - SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), + IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey(), diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/MappingStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/MappingStatsTests.java index 96954458c18c4..0617053769da0 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/MappingStatsTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/MappingStatsTests.java @@ -17,8 +17,8 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.script.Script; import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.test.AbstractWireSerializingTestCase; @@ -586,7 +586,7 @@ public void testSourceModes() { int numDisabledIndices = randomIntBetween(1, 5); for (int i = 0; i < numSyntheticIndices; i++) { IndexMetadata.Builder indexMetadata = new IndexMetadata.Builder("foo-synthetic-" + i).settings( - indexSettings(IndexVersion.current(), 4, 1).put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "synthetic") + indexSettings(IndexVersion.current(), 4, 1).put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "synthetic") ); builder.put(indexMetadata); } @@ -594,7 +594,7 @@ public void testSourceModes() { IndexMetadata.Builder indexMetadata; if (randomBoolean()) { indexMetadata = new IndexMetadata.Builder("foo-stored-" + i).settings( - indexSettings(IndexVersion.current(), 4, 1).put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "stored") + indexSettings(IndexVersion.current(), 4, 1).put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "stored") ); } else { indexMetadata = new IndexMetadata.Builder("foo-stored-" + i).settings(indexSettings(IndexVersion.current(), 4, 1)); @@ -603,7 +603,7 @@ public void testSourceModes() { } for (int i = 0; i < numDisabledIndices; i++) { IndexMetadata.Builder indexMetadata = new IndexMetadata.Builder("foo-disabled-" + i).settings( - indexSettings(IndexVersion.current(), 4, 1).put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "disabled") + indexSettings(IndexVersion.current(), 4, 1).put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "disabled") ); builder.put(indexMetadata); } diff --git a/server/src/test/java/org/elasticsearch/index/engine/LuceneSyntheticSourceChangesSnapshotTests.java b/server/src/test/java/org/elasticsearch/index/engine/LuceneSyntheticSourceChangesSnapshotTests.java index 2a6c3428d6d45..a5d5d9b210e33 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/LuceneSyntheticSourceChangesSnapshotTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/LuceneSyntheticSourceChangesSnapshotTests.java @@ -19,14 +19,12 @@ import java.io.IOException; -import static org.elasticsearch.index.mapper.SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING; - public class LuceneSyntheticSourceChangesSnapshotTests extends SearchBasedChangesSnapshotTests { @Override protected Settings indexSettings() { return Settings.builder() .put(super.indexSettings()) - .put(INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.name()) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.name()) .put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), true) .build(); } diff --git a/server/src/test/java/org/elasticsearch/index/engine/TranslogOperationAsserterTests.java b/server/src/test/java/org/elasticsearch/index/engine/TranslogOperationAsserterTests.java index b764bce464d15..d0455c14bd784 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/TranslogOperationAsserterTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/TranslogOperationAsserterTests.java @@ -24,8 +24,6 @@ import java.io.IOException; -import static org.elasticsearch.index.mapper.SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING; - public class TranslogOperationAsserterTests extends EngineTestCase { @Override @@ -33,7 +31,7 @@ protected Settings indexSettings() { return Settings.builder() .put(super.indexSettings()) .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) - .put(INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.name()) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.name()) .put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), true) .build(); } @@ -57,10 +55,10 @@ EngineConfig engineConfig(boolean useSyntheticSource) { EngineConfig config = engine.config(); Settings.Builder settings = Settings.builder().put(config.getIndexSettings().getSettings()); if (useSyntheticSource) { - settings.put(INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.name()); + settings.put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.name()); settings.put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), true); } else { - settings.put(INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.name()); + settings.put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.name()); settings.put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), false); } IndexMetadata imd = IndexMetadata.builder(config.getIndexSettings().getIndexMetadata()).settings(settings).build(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java index 8646e1b66dcb0..1ba5f423d4b03 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java @@ -12,6 +12,7 @@ import org.apache.lucene.index.DirectoryReader; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; @@ -130,7 +131,7 @@ private MapperService mapperServiceWithCustomSettings( for (var entry : customSettings.entrySet()) { settings.put(entry.getKey(), entry.getValue()); } - settings.put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC); + settings.put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC); return createMapperService(settings.build(), mapping(mapping)); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java index b7693513a434d..bc560d94b8f52 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java @@ -427,7 +427,7 @@ public void testRecoverySourceWitInvalidSettings() { { Settings settings = Settings.builder() - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.toString()) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.toString()) .put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), true) .build(); IllegalArgumentException exc = expectThrows( @@ -470,7 +470,7 @@ public void testRecoverySourceWitInvalidSettings() { public void testRecoverySourceWithSyntheticSource() throws IOException { { Settings settings = Settings.builder() - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.toString()) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.toString()) .build(); MapperService mapperService = createMapperService(settings, topMapping(b -> {})); DocumentMapper docMapper = mapperService.documentMapper(); @@ -480,7 +480,7 @@ public void testRecoverySourceWithSyntheticSource() throws IOException { } { Settings settings = Settings.builder() - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.toString()) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.toString()) .put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), true) .build(); MapperService mapperService = createMapperService(settings, topMapping(b -> {})); @@ -539,7 +539,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { final XContentBuilder mappings = topMapping(b -> {}); final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name()) - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) .build(); final MapperService mapperService = createMapperService(settings, mappings); DocumentMapper docMapper = mapperService.documentMapper(); @@ -549,7 +549,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { final XContentBuilder mappings = topMapping(b -> {}); final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name()) - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED) .build(); final MapperService mapperService = createMapperService(settings, mappings); final DocumentMapper docMapper = mapperService.documentMapper(); @@ -559,7 +559,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { final XContentBuilder mappings = topMapping(b -> {}); final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name()) - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED) .build(); final MapperService mapperService = createMapperService(settings, mappings); final DocumentMapper docMapper = mapperService.documentMapper(); @@ -571,7 +571,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { final XContentBuilder mappings = topMapping(b -> {}); final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) .build(); final MapperService mapperService = createMapperService(settings, mappings); DocumentMapper docMapper = mapperService.documentMapper(); @@ -581,7 +581,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { final XContentBuilder mappings = topMapping(b -> {}); final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED) .build(); final MapperService mapperService = createMapperService(settings, mappings); final DocumentMapper docMapper = mapperService.documentMapper(); @@ -591,7 +591,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { final XContentBuilder mappings = topMapping(b -> {}); final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED) .build(); var ex = expectThrows(MapperParsingException.class, () -> createMapperService(settings, mappings)); assertEquals("Failed to parse mapping: _source can not be disabled in index using [logsdb] index mode", ex.getMessage()); @@ -613,7 +613,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { """; final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name()) - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing_field") .build(); final MapperService mapperService = createMapperService(settings, mappings); @@ -635,7 +635,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { """; final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name()) - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing_field") .build(); final MapperService mapperService = createMapperService(settings, mappings); @@ -657,7 +657,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { """; final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name()) - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing_field") .build(); var ex = expectThrows(MapperParsingException.class, () -> createMapperService(settings, mappings)); @@ -668,7 +668,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { { final XContentBuilder mappings = topMapping(b -> {}); final Settings settings = Settings.builder() - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) .build(); final MapperService mapperService = createMapperService(settings, mappings); DocumentMapper docMapper = mapperService.documentMapper(); @@ -677,7 +677,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { { final XContentBuilder mappings = topMapping(b -> {}); final Settings settings = Settings.builder() - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED) .build(); final MapperService mapperService = createMapperService(settings, mappings); final DocumentMapper docMapper = mapperService.documentMapper(); @@ -686,7 +686,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { { final XContentBuilder mappings = topMapping(b -> {}); final Settings settings = Settings.builder() - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED) .build(); final MapperService mapperService = createMapperService(settings, mappings); final DocumentMapper docMapper = mapperService.documentMapper(); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index bce88a2a8d2f7..b33abc2d781e8 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -1180,7 +1180,7 @@ private void assertSyntheticSourceWithTranslogSnapshot(SyntheticSourceSupport su var firstExample = support.example(1); int maxDocs = randomIntBetween(20, 50); var settings = Settings.builder() - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) .put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), true) .build(); var mapperService = createMapperService(getVersion(), settings, () -> true, mapping(b -> { diff --git a/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java b/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java index 0bb4afe51b85a..e8e19bad2a7ef 100644 --- a/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java +++ b/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java @@ -368,7 +368,7 @@ public void testSyntheticSource() throws Exception { if ("leader".equals(targetCluster)) { logger.info("Running against leader cluster"); Settings settings = Settings.builder() - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) .build(); createIndex(adminClient(), leaderIndexName, settings, """ "properties": {"kwd": {"type": "keyword"}}}""", null); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java index 357e1bca38e8f..6bc25215855ee 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java @@ -18,7 +18,6 @@ import org.elasticsearch.index.MapperTestUtils; import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.ccr.Ccr; import org.elasticsearch.xpack.ccr.CcrSettings; @@ -335,7 +334,7 @@ public void testDynamicIndexSettingsAreClassified() { replicatedSettings.add(IndexSettings.PREFER_ILM_SETTING); replicatedSettings.add(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING); replicatedSettings.add(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING); - replicatedSettings.add(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING); + replicatedSettings.add(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING); for (Setting setting : IndexScopedSettings.BUILT_IN_INDEX_SETTINGS) { // removed settings have no effect, they are only there for BWC diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotShardTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotShardTests.java index 81d194ce84131..a2cdfff2e2c80 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotShardTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotShardTests.java @@ -46,6 +46,7 @@ import org.elasticsearch.core.IOUtils; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.engine.Engine; @@ -53,7 +54,6 @@ import org.elasticsearch.index.engine.InternalEngineFactory; import org.elasticsearch.index.fieldvisitor.FieldsVisitor; import org.elasticsearch.index.mapper.SeqNoFieldMapper; -import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.index.seqno.RetentionLeaseSyncer; import org.elasticsearch.index.seqno.SeqNoStats; @@ -162,7 +162,7 @@ public void testSourceIncompleteSyntheticSourceNoDoc() throws IOException { .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "synthetic") + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "synthetic") .build(); IndexMetadata metadata = IndexMetadata.builder(shardRouting.getIndexName()).settings(settings).primaryTerm(0, primaryTerm).build(); IndexShard shard = newShard(shardRouting, metadata, null, new InternalEngineFactory()); diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBUsageTransportAction.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBUsageTransportAction.java index f4fa2a29d79a0..cdf02273b9df5 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBUsageTransportAction.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBUsageTransportAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.features.FeatureService; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.monitor.metrics.IndexModeStatsActionType; @@ -29,8 +30,6 @@ import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction; import org.elasticsearch.xpack.core.application.LogsDBFeatureSetUsage; -import static org.elasticsearch.index.mapper.SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING; - public class LogsDBUsageTransportAction extends XPackUsageFeatureTransportAction { private final ClusterService clusterService; private final FeatureService featureService; @@ -71,7 +70,7 @@ protected void masterOperation( for (IndexMetadata indexMetadata : state.metadata()) { if (indexMetadata.getIndexMode() == IndexMode.LOGSDB) { numIndices++; - if (INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexMetadata.getSettings()) == SourceFieldMapper.Mode.SYNTHETIC) { + if (IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexMetadata.getSettings()) == SourceFieldMapper.Mode.SYNTHETIC) { numIndicesWithSyntheticSources++; } } diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java index 3316ac3603e9e..633dd6123c088 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java @@ -117,7 +117,7 @@ && matchesLogsPattern(dataStreamName)) { if (settingsBuilder == null) { settingsBuilder = Settings.builder(); } - settingsBuilder.put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.toString()); + settingsBuilder.put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.toString()); } } @@ -206,7 +206,7 @@ MappingHints getMappingHints( var tmpIndexMetadata = buildIndexMetadataForMapperService(indexName, templateIndexMode, indexTemplateAndCreateRequestSettings); var indexMode = tmpIndexMetadata.getIndexMode(); boolean hasSyntheticSourceUsage = false; - if (SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.exists(tmpIndexMetadata.getSettings()) + if (IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.exists(tmpIndexMetadata.getSettings()) || indexMode == IndexMode.LOGSDB || indexMode == IndexMode.TIME_SERIES) { // In case when index mode is tsdb or logsdb and only _source.mode mapping attribute is specified, then the default @@ -214,7 +214,7 @@ MappingHints getMappingHints( // then configuring the index.mapping.source.mode setting to stored has no effect. Additionally _source.mode can't be set // to disabled, because that isn't allowed with logsdb/tsdb. In other words setting index.mapping.source.mode setting to // stored when _source.mode mapping attribute is stored is fine as it has no effect, but avoids creating MapperService. - var sourceMode = SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(tmpIndexMetadata.getSettings()); + var sourceMode = IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(tmpIndexMetadata.getSettings()); hasSyntheticSourceUsage = sourceMode == SourceFieldMapper.Mode.SYNTHETIC; if (IndexSortConfig.INDEX_SORT_FIELD_SETTING.get(indexTemplateAndCreateRequestSettings).isEmpty() == false) { // Custom sort config, no point for further checks on [host.name] field. diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LegacyLicenceIntegrationTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LegacyLicenceIntegrationTests.java index f8f307b572f33..b2a533f6b76c9 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LegacyLicenceIntegrationTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LegacyLicenceIntegrationTests.java @@ -9,7 +9,7 @@ import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.mapper.SourceFieldMapper; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.license.AbstractLicensesIntegrationTestCase; import org.elasticsearch.license.GetFeatureUsageRequest; import org.elasticsearch.license.GetFeatureUsageResponse; @@ -104,11 +104,11 @@ public void testSyntheticSourceUsageTracksBothLegacyAndRegularFeature() throws E } private void createIndexWithSyntheticSourceAndAssertExpectedType(String indexName, String expectedType) { - var settings = Settings.builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "synthetic").build(); + var settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "synthetic").build(); createIndex(indexName, settings); var response = admin().indices().getSettings(new GetSettingsRequest().indices(indexName)).actionGet(); assertThat( - response.getIndexToSettings().get(indexName).get(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey()), + response.getIndexToSettings().get(indexName).get(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey()), equalTo(expectedType) ); } diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java index 71d52842ecbae..77319a881f1e5 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java @@ -640,7 +640,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() throws ); Metadata metadata = mb.build(); LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(false); - Settings settings = builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) + Settings settings = builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) .build(); Settings result = provider.getAdditionalIndexSettings( @@ -666,7 +666,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() throws List.of() ); assertThat(result.size(), equalTo(1)); - assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); + assertEquals(SourceFieldMapper.Mode.STORED, IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); assertThat(newMapperServiceCounter.get(), equalTo(2)); result = provider.getAdditionalIndexSettings( @@ -679,7 +679,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() throws List.of() ); assertThat(result.size(), equalTo(1)); - assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); + assertEquals(SourceFieldMapper.Mode.STORED, IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); assertThat(newMapperServiceCounter.get(), equalTo(3)); result = provider.getAdditionalIndexSettings( @@ -692,7 +692,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() throws List.of() ); assertThat(result.size(), equalTo(3)); - assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); + assertEquals(SourceFieldMapper.Mode.STORED, IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertTrue(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertThat(newMapperServiceCounter.get(), equalTo(4)); @@ -747,7 +747,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSourceFileMatch( List.of() ); assertThat(result.size(), equalTo(4)); - assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); + assertEquals(SourceFieldMapper.Mode.STORED, IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); assertEquals(IndexMode.LOGSDB, IndexSettings.MODE.get(result)); assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertTrue(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java index 4921344278845..7fa2f11880f40 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java @@ -10,9 +10,9 @@ import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.MapperTestUtils; -import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.license.License; import org.elasticsearch.license.LicenseService; import org.elasticsearch.license.XPackLicenseState; @@ -58,16 +58,16 @@ public void setup() throws Exception { } public void testGetAdditionalIndexSettingsDefault() { - Settings settings = Settings.builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); + Settings settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); String dataStreamName = "metrics-my-app"; String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); var result = provider.getAdditionalIndexSettings(indexName, dataStreamName, null, null, null, settings, List.of()); assertThat(result.size(), equalTo(1)); - assertThat(result.get(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey()), equalTo("STORED")); + assertThat(result.get(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey()), equalTo("STORED")); } public void testGetAdditionalIndexSettingsApm() throws IOException { - Settings settings = Settings.builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); + Settings settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); String dataStreamName = "metrics-apm.app.test"; String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); var result = provider.getAdditionalIndexSettings(indexName, dataStreamName, null, null, null, settings, List.of()); @@ -75,7 +75,7 @@ public void testGetAdditionalIndexSettingsApm() throws IOException { } public void testGetAdditionalIndexSettingsProfiling() throws IOException { - Settings settings = Settings.builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); + Settings settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); for (String dataStreamName : new String[] { "profiling-metrics", "profiling-events" }) { String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); var result = provider.getAdditionalIndexSettings(indexName, dataStreamName, null, null, null, settings, List.of()); @@ -89,7 +89,7 @@ public void testGetAdditionalIndexSettingsProfiling() throws IOException { } public void testGetAdditionalIndexSettingsTsdb() throws IOException { - Settings settings = Settings.builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); + Settings settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); String dataStreamName = "metrics-my-app"; String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); var result = provider.getAdditionalIndexSettings(indexName, dataStreamName, IndexMode.TIME_SERIES, null, null, settings, List.of()); @@ -118,11 +118,11 @@ public void testGetAdditionalIndexSettingsTsdbAfterCutoffDate() throws Exception true ); - Settings settings = Settings.builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); + Settings settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); String dataStreamName = "metrics-my-app"; String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); var result = provider.getAdditionalIndexSettings(indexName, dataStreamName, IndexMode.TIME_SERIES, null, null, settings, List.of()); assertThat(result.size(), equalTo(1)); - assertThat(result.get(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey()), equalTo("STORED")); + assertThat(result.get(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey()), equalTo("STORED")); } } From 8114f85bec046400a0f0a8f957f07186e7f0d9c4 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 15 Jan 2025 01:35:19 +1100 Subject: [PATCH 22/31] Mute org.elasticsearch.repositories.blobstore.testkit.analyze.MinioRepositoryAnalysisRestIT org.elasticsearch.repositories.blobstore.testkit.analyze.MinioRepositoryAnalysisRestIT #118548 --- muted-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 9cf2ef065f82f..993fa849dcd31 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -273,6 +273,8 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/120115 - class: org.elasticsearch.repositories.s3.RepositoryS3MinioBasicCredentialsRestIT issue: https://github.com/elastic/elasticsearch/issues/120117 +- class: org.elasticsearch.repositories.blobstore.testkit.analyze.MinioRepositoryAnalysisRestIT + issue: https://github.com/elastic/elasticsearch/issues/118548 # Examples: # From 7ddcfeb638fad5d8d4d77d8097d4ca264ca79dea Mon Sep 17 00:00:00 2001 From: David Turner Date: Tue, 14 Jan 2025 15:07:58 +0000 Subject: [PATCH 23/31] Fix `KibanaThreadPoolIT` (#120097) This test will fail to see the rejection if the GET retries on a different shard copy, so with this commit we make sure there are no spare shard copies on which to retry. Closes #113939 --- .../elasticsearch/kibana/KibanaThreadPoolIT.java | 16 ++++++++++------ muted-tests.yml | 3 --- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/modules/kibana/src/internalClusterTest/java/org/elasticsearch/kibana/KibanaThreadPoolIT.java b/modules/kibana/src/internalClusterTest/java/org/elasticsearch/kibana/KibanaThreadPoolIT.java index 553e4696af316..a9ab0c02612f6 100644 --- a/modules/kibana/src/internalClusterTest/java/org/elasticsearch/kibana/KibanaThreadPoolIT.java +++ b/modules/kibana/src/internalClusterTest/java/org/elasticsearch/kibana/KibanaThreadPoolIT.java @@ -16,6 +16,8 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor; @@ -23,7 +25,6 @@ import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPoolStats; @@ -49,10 +50,6 @@ * threads that wait on a phaser. This lets us verify that operations on system indices * are being directed to other thread pools.

*/ -@TestLogging( - reason = "investigate", - value = "org.elasticsearch.kibana.KibanaThreadPoolIT:DEBUG,org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor:TRACE" -) public class KibanaThreadPoolIT extends ESIntegTestCase { private static final Logger logger = LogManager.getLogger(KibanaThreadPoolIT.class); @@ -68,6 +65,8 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { .put("thread_pool.write.queue_size", 1) .put("thread_pool.get.size", 1) .put("thread_pool.get.queue_size", 1) + // a rejected GET may retry on an INITIALIZING shard (the target of a relocation) and unexpectedly succeed, so block rebalancing + .put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE) .build(); } @@ -112,7 +111,12 @@ public void testKibanaThreadPoolByPassesBlockedThreadPools() throws Exception { } public void testBlockedThreadPoolsRejectUserRequests() throws Exception { - assertAcked(client().admin().indices().prepareCreate(USER_INDEX)); + assertAcked( + client().admin() + .indices() + .prepareCreate(USER_INDEX) + .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)) // avoid retrying rejected actions + ); runWithBlockedThreadPools(this::assertThreadPoolsBlocked); diff --git a/muted-tests.yml b/muted-tests.yml index 993fa849dcd31..52227538d92b2 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -52,9 +52,6 @@ tests: - class: org.elasticsearch.xpack.transform.integration.TransformIT method: testStopWaitForCheckpoint issue: https://github.com/elastic/elasticsearch/issues/106113 -- class: org.elasticsearch.kibana.KibanaThreadPoolIT - method: testBlockedThreadPoolsRejectUserRequests - issue: https://github.com/elastic/elasticsearch/issues/113939 - class: org.elasticsearch.xpack.inference.TextEmbeddingCrudIT method: testPutE5Small_withPlatformAgnosticVariant issue: https://github.com/elastic/elasticsearch/issues/113983 From 7750cf5b94e05bf7ee1c5c5ce796d62b04407d88 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Tue, 14 Jan 2025 07:15:24 -0800 Subject: [PATCH 24/31] Remove default auto_expand_replicas of lookup indices (#120073) This change disables auto_expand_replicas on lookup indices to enhance the lookup join user experience. Users can, however, enable this setting at any time to optimize performance. --- rest-api-spec/build.gradle | 2 ++ .../resources/rest-api-spec/test/indices.create/10_basic.yml | 2 -- .../java/org/elasticsearch/index/LookupIndexModeIT.java | 3 +-- server/src/main/java/org/elasticsearch/index/IndexMode.java | 5 +---- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 1da8e906582b1..e4b46b98cedda 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -85,4 +85,6 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task -> task.skipTest("search.vectors/110_knn_query_with_filter/PRE_FILTER: pre-filter across multiple aliases", "waiting for #118774 backport") task.skipTest("search.vectors/160_knn_query_missing_params/kNN search in a dis_max query - missing num_candidates", "waiting for #118774 backport") task.skipTest("search.highlight/30_max_analyzed_offset/Plain highlighter with max_analyzed_offset < 0 should FAIL", "semantics of test has changed") + task.skipTest("indices.create/10_basic/Create lookup index", "default auto_expand_replicas was removed") + task.skipTest("indices.create/10_basic/Create lookup index with one shard", "default auto_expand_replicas was removed") }) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/10_basic.yml index d0e1759073e1b..8645c91a51ad3 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/10_basic.yml @@ -171,7 +171,6 @@ index: test_lookup - match: { test_lookup.settings.index.number_of_shards: "1"} - - match: { test_lookup.settings.index.auto_expand_replicas: "0-all"} --- "Create lookup index with one shard": @@ -196,7 +195,6 @@ index: test_lookup - match: { test_lookup.settings.index.number_of_shards: "1"} - - match: { test_lookup.settings.index.auto_expand_replicas: "0-all"} --- "Create lookup index with two shards": diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/LookupIndexModeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/LookupIndexModeIT.java index 960ee2fd7ca60..6bca87ebd6e3d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/LookupIndexModeIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/LookupIndexModeIT.java @@ -49,7 +49,7 @@ public void testBasic() { assertAcked(client().admin().indices().execute(TransportCreateIndexAction.TYPE, createRequest)); Settings settings = client().admin().indices().prepareGetSettings("hosts").get().getIndexToSettings().get("hosts"); assertThat(settings.get("index.mode"), equalTo("lookup")); - assertThat(settings.get("index.auto_expand_replicas"), equalTo("0-all")); + assertNull(settings.get("index.auto_expand_replicas")); Map allHosts = Map.of( "192.168.1.2", "Windows", @@ -141,7 +141,6 @@ public void testResizeLookupIndex() { Settings settings = client().admin().indices().prepareGetSettings("lookup-2").get().getIndexToSettings().get("lookup-2"); assertThat(settings.get("index.mode"), equalTo("lookup")); assertThat(settings.get("index.number_of_shards"), equalTo("1")); - assertThat(settings.get("index.auto_expand_replicas"), equalTo("0-all")); ResizeRequest split = new ResizeRequest("lookup-3", "lookup-1"); split.setResizeType(ResizeType.SPLIT); diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index f0814577bd203..7287a0bf307b9 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -623,10 +623,7 @@ public Settings getAdditionalIndexSettings( } } if (indexMode == LOOKUP) { - return Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-all") - .build(); + return Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).build(); } else { return Settings.EMPTY; } From 187a6a7dc50faa112f8477e0c8633dbde1493051 Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Tue, 14 Jan 2025 10:17:16 -0500 Subject: [PATCH 25/31] ESQL Support IN operator for Date nanos (#119772) Add support for using nanosecond dates with the IN operator. This behavior should be consistent with equals, and support comparisons between milliseconds and nanoseconds the same as the binary comparison operators support it. Resolves #118578 --------- Co-authored-by: elasticsearchmachine --- docs/changelog/119772.yaml | 6 + x-pack/plugin/esql/build.gradle | 27 ++- .../src/main/resources/date.csv-spec | 44 ++++ .../src/main/resources/date_nanos.csv-spec | 65 ++++++ .../comparison/InMillisNanosEvaluator.java | 189 ++++++++++++++++++ .../comparison/InNanosMillisEvaluator.java | 189 ++++++++++++++++++ .../xpack/esql/action/EsqlCapabilities.java | 5 + .../predicate/operator/comparison/In.java | 104 ++++++++-- .../operator/comparison/X-InEvaluator.java.st | 19 +- 9 files changed, 620 insertions(+), 28 deletions(-) create mode 100644 docs/changelog/119772.yaml create mode 100644 x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InMillisNanosEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InNanosMillisEvaluator.java diff --git a/docs/changelog/119772.yaml b/docs/changelog/119772.yaml new file mode 100644 index 0000000000000..58d483566b109 --- /dev/null +++ b/docs/changelog/119772.yaml @@ -0,0 +1,6 @@ +pr: 119772 +summary: ESQL Support IN operator for Date nanos +area: ES|QL +type: enhancement +issues: + - 118578 diff --git a/x-pack/plugin/esql/build.gradle b/x-pack/plugin/esql/build.gradle index 9cbcd8d49b586..d7fa786a24e7c 100644 --- a/x-pack/plugin/esql/build.gradle +++ b/x-pack/plugin/esql/build.gradle @@ -283,8 +283,9 @@ tasks.named('checkstyleMain').configure { exclude { normalize(it.file.toString()).contains("src/main/generated") } } -def prop(Type, type, TYPE, BYTES, Array) { +def prop(Name, Type, type, TYPE, BYTES, Array) { return [ + "Name" : Name, "Type" : Type, "type" : type, "TYPE" : TYPE, @@ -296,15 +297,19 @@ def prop(Type, type, TYPE, BYTES, Array) { "double" : type == "double" ? "true" : "", "BytesRef" : type == "BytesRef" ? "true" : "", "boolean" : type == "boolean" ? "true" : "", + "nanosMillis" : Name == "NanosMillis" ? "true" : "", + "millisNanos" : Name == "MillisNanos" ? "true" : "", ] } tasks.named('stringTemplates').configure { - var intProperties = prop("Int", "int", "INT", "Integer.BYTES", "IntArray") - var longProperties = prop("Long", "long", "LONG", "Long.BYTES", "LongArray") - var doubleProperties = prop("Double", "double", "DOUBLE", "Double.BYTES", "DoubleArray") - var bytesRefProperties = prop("BytesRef", "BytesRef", "BYTES_REF", "org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_REF", "") - var booleanProperties = prop("Boolean", "boolean", "BOOLEAN", "Byte.BYTES", "BitArray") + var intProperties = prop("Int", "Int", "int", "INT", "Integer.BYTES", "IntArray") + var longProperties = prop("Long", "Long", "long", "LONG", "Long.BYTES", "LongArray") + var nanosMillisProperties = prop("NanosMillis", "Long", "long", "LONG", "Long.BYTES", "LongArray") + var millisNanosProperties = prop("MillisNanos", "Long", "long", "LONG", "Long.BYTES", "LongArray") + var doubleProperties = prop("Double", "Double", "double", "DOUBLE", "Double.BYTES", "DoubleArray") + var bytesRefProperties = prop("BytesRef", "BytesRef", "BytesRef", "BYTES_REF", "org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_REF", "") + var booleanProperties = prop("Boolean", "Boolean", "boolean", "BOOLEAN", "Byte.BYTES", "BitArray") File inInputFile = file("src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/X-InEvaluator.java.st") template { @@ -322,6 +327,16 @@ tasks.named('stringTemplates').configure { it.inputFile = inInputFile it.outputFile = "org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InLongEvaluator.java" } + template { + it.properties = nanosMillisProperties + it.inputFile = inInputFile + it.outputFile = "org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InNanosMillisEvaluator.java" + } + template { + it.properties = millisNanosProperties + it.inputFile = inInputFile + it.outputFile = "org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InMillisNanosEvaluator.java" + } template { it.properties = doubleProperties it.inputFile = inInputFile diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec index 57c4fca6223e1..bb46c9d31f745 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec @@ -1450,3 +1450,47 @@ FROM employees cnt:long 19 ; + +implicit casting strings to dates for IN operator +FROM employees +| WHERE birth_date IN ("1953-04-20", "1958-10-31") +| KEEP emp_no, first_name; +ignoreOrder:true + +emp_no:integer | first_name:keyword +10006 | Anneke +10025 | Prasadram +; + +IN operator with null in list, finds match + +FROM employees +| EVAL x = NULL +| WHERE birth_date IN (TO_DATETIME("1958-02-19T00:00:00Z"), x) +| KEEP birth_date, first_name; + +birth_date:datetime | first_name:keyword +1958-02-19T00:00:00.000Z | Saniya +; + +IN operator with null in list, doesn't find match + +FROM employees +| EVAL x = NULL +| WHERE birth_date IN (TO_DATETIME("1900-02-19T00:00:00Z"), x) +| KEEP birth_date, first_name; + +birth_date:datetime | first_name:keyword +; + +IN operator with null in list, doesn't find match, EVAL to check value + +FROM employees +| EVAL x = NULL +| EVAL result = birth_date IN (TO_DATETIME("1900-02-19T00:00:00Z"), x) +| LIMIT 1 +| KEEP result; + +result:boolean +null +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec index 4206d6b48699f..71a65f059528a 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec @@ -1161,3 +1161,68 @@ row dt = to_date_nanos(0::long) plus:date_nanos 1970-01-01T00:00:01.000Z ; + +Date Nanos IN constant date nanos +required_capability: date_nanos_in_operator +required_capability: to_date_nanos + +FROM date_nanos +| WHERE MV_FIRST(nanos) IN (TO_DATE_NANOS("2023-10-23T13:55:01.543123456Z"), TO_DATE_NANOS("2023-10-23T12:27:28.948Z"), TO_DATE_NANOS("2017-10-23T13:53:55.832987654Z")); +ignoreOrder:true + +millis:date | nanos:date_nanos | num:long +2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456 +2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000 +; + +Date Nanos IN constant date nanos, implicit casting +required_capability: date_nanos_in_operator +required_capability: to_date_nanos +required_capability: date_nanos_implicit_casting + +FROM date_nanos +| WHERE MV_FIRST(nanos) IN ("2023-10-23T13:55:01.543123456Z", "2023-10-23T12:27:28.948Z", "2017-10-23T13:53:55.832987654Z"); +ignoreOrder:true + +millis:date | nanos:date_nanos | num:long +2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456 +2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000 +; + +Date Nanos IN date nanos field, implicit casting +required_capability: date_nanos_in_operator +required_capability: to_date_nanos +required_capability: date_nanos_implicit_casting + +FROM date_nanos +| WHERE "2023-10-23T13:55:01.543123456Z" IN (MV_FIRST(nanos)); + +millis:date | nanos:date_nanos | num:long +2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456 +; + +Date nanos IN millisecond date field +required_capability: date_nanos_in_operator +required_capability: to_date_nanos +required_capability: date_nanos_implicit_casting + +# Note: It is important to have two dates in the IN so the optimizer doesn't turn this into an == +FROM date_nanos +| WHERE MV_FIRST(nanos) IN (TO_DATETIME("2023-10-23T12:27:28.948"), TO_DATETIME("2020-02-02")); + +millis:date | nanos:date_nanos | num:long +2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000 +; + +Millisecond field IN date nanos constants +required_capability: date_nanos_in_operator +required_capability: to_date_nanos +required_capability: date_nanos_implicit_casting + +# Note: It is important to have two dates in the IN so the optimizer doesn't turn this into an == +FROM date_nanos +| WHERE MV_FIRST(millis) IN (TO_DATE_NANOS("2023-10-23T12:27:28.948"), TO_DATE_NANOS("2020-02-02")); + +millis:date | nanos:date_nanos | num:long +2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000 +; diff --git a/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InMillisNanosEvaluator.java b/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InMillisNanosEvaluator.java new file mode 100644 index 0000000000000..14ca1e9a768fb --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InMillisNanosEvaluator.java @@ -0,0 +1,189 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; + +import java.util.Arrays; +import java.util.BitSet; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link In}. + * This class is generated. Edit {@code X-InEvaluator.java.st} instead. + */ +public class InMillisNanosEvaluator implements EvalOperator.ExpressionEvaluator { + private final Source source; + + private final EvalOperator.ExpressionEvaluator lhs; + + private final EvalOperator.ExpressionEvaluator[] rhs; + + private final DriverContext driverContext; + + private Warnings warnings; + + public InMillisNanosEvaluator( + Source source, + EvalOperator.ExpressionEvaluator lhs, + EvalOperator.ExpressionEvaluator[] rhs, + DriverContext driverContext + ) { + this.source = source; + this.lhs = lhs; + this.rhs = rhs; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (LongBlock lhsBlock = (LongBlock) lhs.eval(page)) { + LongBlock[] rhsBlocks = new LongBlock[rhs.length]; + try (Releasable rhsRelease = Releasables.wrap(rhsBlocks)) { + for (int i = 0; i < rhsBlocks.length; i++) { + rhsBlocks[i] = (LongBlock) rhs[i].eval(page); + } + LongVector lhsVector = lhsBlock.asVector(); + if (lhsVector == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlocks); + } + LongVector[] rhsVectors = new LongVector[rhs.length]; + for (int i = 0; i < rhsBlocks.length; i++) { + rhsVectors[i] = rhsBlocks[i].asVector(); + if (rhsVectors[i] == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlocks); + } + } + return eval(page.getPositionCount(), lhsVector, rhsVectors); + } + } + } + + private BooleanBlock eval(int positionCount, LongBlock lhsBlock, LongBlock[] rhsBlocks) { + try (BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + long[] rhsValues = new long[rhs.length]; + BitSet nulls = new BitSet(rhs.length); + BitSet mvs = new BitSet(rhs.length); + boolean foundMatch; + for (int p = 0; p < positionCount; p++) { + if (lhsBlock.isNull(p)) { + result.appendNull(); + continue; + } + if (lhsBlock.getValueCount(p) != 1) { + if (lhsBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue; + } + // unpack rhsBlocks into rhsValues + nulls.clear(); + mvs.clear(); + for (int i = 0; i < rhsBlocks.length; i++) { + if (rhsBlocks[i].isNull(p)) { + nulls.set(i); + continue; + } + if (rhsBlocks[i].getValueCount(p) > 1) { + mvs.set(i); + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + continue; + } + int o = rhsBlocks[i].getFirstValueIndex(p); + rhsValues[i] = rhsBlocks[i].getLong(o); + } + if (nulls.cardinality() == rhsBlocks.length || mvs.cardinality() == rhsBlocks.length) { + result.appendNull(); + continue; + } + foundMatch = In.process(nulls, mvs, lhsBlock.getLong(lhsBlock.getFirstValueIndex(p)), rhsValues); + if (foundMatch) { + result.appendBoolean(true); + } else { + if (nulls.cardinality() > 0) { + result.appendNull(); + } else { + result.appendBoolean(false); + } + } + } + return result.build(); + } + } + + private BooleanBlock eval(int positionCount, LongVector lhsVector, LongVector[] rhsVectors) { + try (BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + long[] rhsValues = new long[rhs.length]; + for (int p = 0; p < positionCount; p++) { + // unpack rhsVectors into rhsValues + for (int i = 0; i < rhsVectors.length; i++) { + rhsValues[i] = rhsVectors[i].getLong(p); + } + result.appendBoolean(In.processMillisNanos(null, null, lhsVector.getLong(p), rhsValues)); + } + return result.build(); + } + } + + @Override + public String toString() { + return "InMillisNanosEvaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(lhs, () -> Releasables.close(rhs)); + } + + private Warnings warnings() { + if (warnings == null) { + this.warnings = Warnings.createWarnings( + driverContext.warningsMode(), + source.source().getLineNumber(), + source.source().getColumnNumber(), + source.text() + ); + } + return warnings; + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + private final EvalOperator.ExpressionEvaluator.Factory lhs; + private final EvalOperator.ExpressionEvaluator.Factory[] rhs; + + Factory(Source source, EvalOperator.ExpressionEvaluator.Factory lhs, EvalOperator.ExpressionEvaluator.Factory[] rhs) { + this.source = source; + this.lhs = lhs; + this.rhs = rhs; + } + + @Override + public InMillisNanosEvaluator get(DriverContext context) { + EvalOperator.ExpressionEvaluator[] rhs = Arrays.stream(this.rhs) + .map(a -> a.get(context)) + .toArray(EvalOperator.ExpressionEvaluator[]::new); + return new InMillisNanosEvaluator(source, lhs.get(context), rhs, context); + } + + @Override + public String toString() { + return "InMillisNanosEvaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InNanosMillisEvaluator.java b/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InNanosMillisEvaluator.java new file mode 100644 index 0000000000000..278ecdba7d9e5 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InNanosMillisEvaluator.java @@ -0,0 +1,189 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; + +import java.util.Arrays; +import java.util.BitSet; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link In}. + * This class is generated. Edit {@code X-InEvaluator.java.st} instead. + */ +public class InNanosMillisEvaluator implements EvalOperator.ExpressionEvaluator { + private final Source source; + + private final EvalOperator.ExpressionEvaluator lhs; + + private final EvalOperator.ExpressionEvaluator[] rhs; + + private final DriverContext driverContext; + + private Warnings warnings; + + public InNanosMillisEvaluator( + Source source, + EvalOperator.ExpressionEvaluator lhs, + EvalOperator.ExpressionEvaluator[] rhs, + DriverContext driverContext + ) { + this.source = source; + this.lhs = lhs; + this.rhs = rhs; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (LongBlock lhsBlock = (LongBlock) lhs.eval(page)) { + LongBlock[] rhsBlocks = new LongBlock[rhs.length]; + try (Releasable rhsRelease = Releasables.wrap(rhsBlocks)) { + for (int i = 0; i < rhsBlocks.length; i++) { + rhsBlocks[i] = (LongBlock) rhs[i].eval(page); + } + LongVector lhsVector = lhsBlock.asVector(); + if (lhsVector == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlocks); + } + LongVector[] rhsVectors = new LongVector[rhs.length]; + for (int i = 0; i < rhsBlocks.length; i++) { + rhsVectors[i] = rhsBlocks[i].asVector(); + if (rhsVectors[i] == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlocks); + } + } + return eval(page.getPositionCount(), lhsVector, rhsVectors); + } + } + } + + private BooleanBlock eval(int positionCount, LongBlock lhsBlock, LongBlock[] rhsBlocks) { + try (BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + long[] rhsValues = new long[rhs.length]; + BitSet nulls = new BitSet(rhs.length); + BitSet mvs = new BitSet(rhs.length); + boolean foundMatch; + for (int p = 0; p < positionCount; p++) { + if (lhsBlock.isNull(p)) { + result.appendNull(); + continue; + } + if (lhsBlock.getValueCount(p) != 1) { + if (lhsBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue; + } + // unpack rhsBlocks into rhsValues + nulls.clear(); + mvs.clear(); + for (int i = 0; i < rhsBlocks.length; i++) { + if (rhsBlocks[i].isNull(p)) { + nulls.set(i); + continue; + } + if (rhsBlocks[i].getValueCount(p) > 1) { + mvs.set(i); + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + continue; + } + int o = rhsBlocks[i].getFirstValueIndex(p); + rhsValues[i] = rhsBlocks[i].getLong(o); + } + if (nulls.cardinality() == rhsBlocks.length || mvs.cardinality() == rhsBlocks.length) { + result.appendNull(); + continue; + } + foundMatch = In.process(nulls, mvs, lhsBlock.getLong(lhsBlock.getFirstValueIndex(p)), rhsValues); + if (foundMatch) { + result.appendBoolean(true); + } else { + if (nulls.cardinality() > 0) { + result.appendNull(); + } else { + result.appendBoolean(false); + } + } + } + return result.build(); + } + } + + private BooleanBlock eval(int positionCount, LongVector lhsVector, LongVector[] rhsVectors) { + try (BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + long[] rhsValues = new long[rhs.length]; + for (int p = 0; p < positionCount; p++) { + // unpack rhsVectors into rhsValues + for (int i = 0; i < rhsVectors.length; i++) { + rhsValues[i] = rhsVectors[i].getLong(p); + } + result.appendBoolean(In.processNanosMillis(null, null, lhsVector.getLong(p), rhsValues)); + } + return result.build(); + } + } + + @Override + public String toString() { + return "InNanosMillisEvaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(lhs, () -> Releasables.close(rhs)); + } + + private Warnings warnings() { + if (warnings == null) { + this.warnings = Warnings.createWarnings( + driverContext.warningsMode(), + source.source().getLineNumber(), + source.source().getColumnNumber(), + source.text() + ); + } + return warnings; + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + private final EvalOperator.ExpressionEvaluator.Factory lhs; + private final EvalOperator.ExpressionEvaluator.Factory[] rhs; + + Factory(Source source, EvalOperator.ExpressionEvaluator.Factory lhs, EvalOperator.ExpressionEvaluator.Factory[] rhs) { + this.source = source; + this.lhs = lhs; + this.rhs = rhs; + } + + @Override + public InNanosMillisEvaluator get(DriverContext context) { + EvalOperator.ExpressionEvaluator[] rhs = Arrays.stream(this.rhs) + .map(a -> a.get(context)) + .toArray(EvalOperator.ExpressionEvaluator[]::new); + return new InNanosMillisEvaluator(source, lhs.get(context), rhs, context); + } + + @Override + public String toString() { + return "InNanosMillisEvaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 999ada6feba03..4531532fac3c5 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -380,6 +380,11 @@ public enum Cap { */ DATE_NANOS_AGGREGATIONS(), + /** + * Support the {@link org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In} operator for date nanos + */ + DATE_NANOS_IN_OPERATOR(), + /** * DATE_PARSE supports reading timezones */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java index 2061c2626aa45..6a35fa8e8b957 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.Vector; import org.elasticsearch.compute.operator.EvalOperator; @@ -41,6 +42,7 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN; import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; +import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS; import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; import static org.elasticsearch.xpack.esql.core.type.DataType.IP; @@ -232,20 +234,47 @@ protected TypeResolution resolveType() { // TODO: move the foldability check fro } DataType dt = value.dataType(); - for (int i = 0; i < list.size(); i++) { - Expression listValue = list.get(i); - if (areCompatible(dt, listValue.dataType()) == false) { - return new TypeResolution( - format( - null, - "{} argument of [{}] must be [{}], found value [{}] type [{}]", - ordinal(i + 1), - sourceText(), - dt.typeName(), - Expressions.name(listValue), - listValue.dataType().typeName() - ) - ); + if (dt.isDate()) { + // If value is a date (nanos or millis), list cannot contain both nanos and millis + DataType seenDateType = null; + for (int i = 0; i < list.size(); i++) { + Expression listValue = list.get(i); + if (seenDateType == null && listValue.dataType().isDate()) { + seenDateType = listValue.dataType(); + } + // The areCompatible test is still necessary to account for nulls. + if ((listValue.dataType().isDate() && listValue.dataType() != seenDateType) + || (listValue.dataType().isDate() == false && areCompatible(dt, listValue.dataType()) == false)) { + return new TypeResolution( + format( + null, + "{} argument of [{}] must be [{}], found value [{}] type [{}]", + ordinal(i + 1), + sourceText(), + dt.typeName(), + Expressions.name(listValue), + listValue.dataType().typeName() + ) + ); + } + } + + } else { + for (int i = 0; i < list.size(); i++) { + Expression listValue = list.get(i); + if (areCompatible(dt, listValue.dataType()) == false) { + return new TypeResolution( + format( + null, + "{} argument of [{}] must be [{}], found value [{}] type [{}]", + ordinal(i + 1), + sourceText(), + dt.typeName(), + Expressions.name(listValue), + listValue.dataType().typeName() + ) + ); + } } } @@ -262,9 +291,19 @@ protected Expression canonicalize() { @Override public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { - var commonType = commonType(); EvalOperator.ExpressionEvaluator.Factory lhs; EvalOperator.ExpressionEvaluator.Factory[] factories; + if (value.dataType() == DATE_NANOS && list.getFirst().dataType() == DATETIME) { + lhs = toEvaluator.apply(value); + factories = list.stream().map(toEvaluator::apply).toArray(EvalOperator.ExpressionEvaluator.Factory[]::new); + return new InNanosMillisEvaluator.Factory(source(), lhs, factories); + } + if (value.dataType() == DATETIME && list.getFirst().dataType() == DATE_NANOS) { + lhs = toEvaluator.apply(value); + factories = list.stream().map(toEvaluator::apply).toArray(EvalOperator.ExpressionEvaluator.Factory[]::new); + return new InMillisNanosEvaluator.Factory(source(), lhs, factories); + } + var commonType = commonType(); if (commonType.isNumeric()) { lhs = Cast.cast(source(), value.dataType(), commonType, toEvaluator.apply(value)); factories = list.stream() @@ -284,7 +323,7 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvalua if (commonType == INTEGER) { return new InIntEvaluator.Factory(source(), lhs, factories); } - if (commonType == LONG || commonType == DATETIME || commonType == UNSIGNED_LONG) { + if (commonType == LONG || commonType == DATETIME || commonType == DATE_NANOS || commonType == UNSIGNED_LONG) { return new InLongEvaluator.Factory(source(), lhs, factories); } if (commonType == KEYWORD @@ -346,6 +385,39 @@ static boolean process(BitSet nulls, BitSet mvs, long lhs, long[] rhs) { return false; } + /** + * Processor for mixed millisecond and nanosecond dates, where the "value" (aka lhs) is in nanoseconds + * and the "list" (aka rhs) is in milliseconds + */ + static boolean processNanosMillis(BitSet nulls, BitSet mvs, long lhs, long[] rhs) { + for (int i = 0; i < rhs.length; i++) { + if ((nulls != null && nulls.get(i)) || (mvs != null && mvs.get(i))) { + continue; + } + if (DateUtils.compareNanosToMillis(lhs, rhs[i]) == 0) { + return true; + } + } + return false; + } + + /** + * Processor for mixed millisecond and nanosecond dates, where the "value" (aka lhs) is in milliseoncds + * and the "list" (aka rhs) is in nanoseconds + */ + static boolean processMillisNanos(BitSet nulls, BitSet mvs, long lhs, long[] rhs) { + for (int i = 0; i < rhs.length; i++) { + if ((nulls != null && nulls.get(i)) || (mvs != null && mvs.get(i))) { + continue; + } + Boolean compResult = DateUtils.compareNanosToMillis(rhs[i], lhs) == 0; + if (compResult == Boolean.TRUE) { + return true; + } + } + return false; + } + static boolean process(BitSet nulls, BitSet mvs, double lhs, double[] rhs) { for (int i = 0; i < rhs.length; i++) { if ((nulls != null && nulls.get(i)) || (mvs != null && mvs.get(i))) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/X-InEvaluator.java.st b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/X-InEvaluator.java.st index f63815f475d8f..0cdea82119cae 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/X-InEvaluator.java.st +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/X-InEvaluator.java.st @@ -42,7 +42,7 @@ import java.util.BitSet; * {@link EvalOperator.ExpressionEvaluator} implementation for {@link In}. * This class is generated. Edit {@code X-InEvaluator.java.st} instead. */ -public class In$Type$Evaluator implements EvalOperator.ExpressionEvaluator { +public class In$Name$Evaluator implements EvalOperator.ExpressionEvaluator { private final Source source; private final EvalOperator.ExpressionEvaluator lhs; @@ -53,7 +53,7 @@ public class In$Type$Evaluator implements EvalOperator.ExpressionEvaluator { private Warnings warnings; - public In$Type$Evaluator( + public In$Name$Evaluator( Source source, EvalOperator.ExpressionEvaluator lhs, EvalOperator.ExpressionEvaluator[] rhs, @@ -166,6 +166,9 @@ $if(boolean)$ foundMatch = lhsBlock.getBoolean(lhsBlock.getFirstValueIndex(p)) ? hasTrue : hasFalse; $elseif(BytesRef)$ foundMatch = In.process(nulls, mvs, lhsBlock.getBytesRef(lhsBlock.getFirstValueIndex(p), lhsScratch), rhsValues); +$elseif(NanosMillis)$ + // Even though these are longs, they're in different units and need special handling + foundMatch = In.processNanosMilils(nulls, mvs, lhsBlock.get$Type$(lhsBlock.getFirstValueIndex(p)), rhsValues); $else$ foundMatch = In.process(nulls, mvs, lhsBlock.get$Type$(lhsBlock.getFirstValueIndex(p)), rhsValues); $endif$ @@ -230,6 +233,10 @@ $if(BytesRef)$ result.appendBoolean(In.process(null, null, lhsVector.getBytesRef(p, lhsScratch), rhsValues)); $elseif(boolean)$ result.appendBoolean(lhsVector.getBoolean(p) ? hasTrue : hasFalse); +$elseif(nanosMillis)$ + result.appendBoolean(In.processNanosMillis(null, null, lhsVector.get$Type$(p), rhsValues)); +$elseif(millisNanos)$ + result.appendBoolean(In.processMillisNanos(null, null, lhsVector.get$Type$(p), rhsValues)); $else$ result.appendBoolean(In.process(null, null, lhsVector.get$Type$(p), rhsValues)); $endif$ @@ -240,7 +247,7 @@ $endif$ @Override public String toString() { - return "In$Type$Evaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]"; + return "In$Name$Evaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]"; } @Override @@ -272,16 +279,16 @@ $endif$ } @Override - public In$Type$Evaluator get(DriverContext context) { + public In$Name$Evaluator get(DriverContext context) { EvalOperator.ExpressionEvaluator[] rhs = Arrays.stream(this.rhs) .map(a -> a.get(context)) .toArray(EvalOperator.ExpressionEvaluator[]::new); - return new In$Type$Evaluator(source, lhs.get(context), rhs, context); + return new In$Name$Evaluator(source, lhs.get(context), rhs, context); } @Override public String toString() { - return "In$Type$Evaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]"; + return "In$Name$Evaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]"; } } } From 30948acd593641611385f55128705bb081ca217f Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 15 Jan 2025 02:23:36 +1100 Subject: [PATCH 26/31] Mute org.elasticsearch.xpack.security.QueryableReservedRolesIT testConfiguredReservedRolesAfterClosingAndOpeningIndex #120127 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 52227538d92b2..5992c3dda800a 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -272,6 +272,9 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/120117 - class: org.elasticsearch.repositories.blobstore.testkit.analyze.MinioRepositoryAnalysisRestIT issue: https://github.com/elastic/elasticsearch/issues/118548 +- class: org.elasticsearch.xpack.security.QueryableReservedRolesIT + method: testConfiguredReservedRolesAfterClosingAndOpeningIndex + issue: https://github.com/elastic/elasticsearch/issues/120127 # Examples: # From 825b5dead429c7ae9fe14e8fb45d731ecbc5cdf3 Mon Sep 17 00:00:00 2001 From: David Turner Date: Tue, 14 Jan 2025 15:34:12 +0000 Subject: [PATCH 27/31] Introduce new handshake version for v9 (#120109) This commit introduces `V9_HANDSHAKE_VERSION` which is the transport version that will eventually be for handshakes sent by v9 nodes. It does not adjust the handshake version yet, because we must first backport this to the v8.18 branch so that v8.18 nodes can understand the new v9 handshakes. --- .../transport/TransportHandshaker.java | 22 ++++-- .../transport/InboundDecoderTests.java | 67 +++++++++++++++---- 2 files changed, 71 insertions(+), 18 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/transport/TransportHandshaker.java b/server/src/main/java/org/elasticsearch/transport/TransportHandshaker.java index a43b1e7440bda..ec9b30e71e62c 100644 --- a/server/src/main/java/org/elasticsearch/transport/TransportHandshaker.java +++ b/server/src/main/java/org/elasticsearch/transport/TransportHandshaker.java @@ -41,9 +41,14 @@ final class TransportHandshaker { * ignores the body of the request. After the handshake, the OutboundHandler uses the min(local,remote) protocol version for all later * messages. * - * This version supports two handshake protocols, v6080099 and v7170099, which respectively have the same message structure as the - * transport protocols of v6.8.0 and v7.17.0. This node only sends v7170099 requests, but it can send a valid response to any v6080099 - * requests that it receives. + * This version supports three handshake protocols, v6080099, v7170099 and v8800000, which respectively have the same message structure + * as the transport protocols of v6.8.0, v7.17.0, and v8.18.0. This node only sends v7170099 requests, but it can send a valid response + * to any v6080099 or v8800000 requests that it receives. + * + * Note that these are not really TransportVersion constants as used elsewhere in ES, they're independent things that just happen to be + * stored in the same location in the message header and which roughly match the same ID numbering scheme. Older versions of ES did + * rely on them matching the real transport protocol (which itself matched the release version numbers), but these days that's no longer + * true. * * Here are some example messages, broken down to show their structure: * @@ -79,7 +84,7 @@ final class TransportHandshaker { * c3 f9 eb 03 -- max acceptable protocol version (vInt: 00000011 11101011 11111001 11000011 == 8060099) * * - * ## v7170099 Request: + * ## v7170099 and v8800000 Requests: * * 45 53 -- 'ES' marker * 00 00 00 31 -- total message length @@ -98,7 +103,7 @@ final class TransportHandshaker { * 04 -- payload length * c3 f9 eb 03 -- max acceptable protocol version (vInt: 00000011 11101011 11111001 11000011 == 8060099) * - * ## v7170099 Response: + * ## v7170099 and v8800000 Responses: * * 45 53 -- 'ES' marker * 00 00 00 17 -- total message length @@ -118,7 +123,12 @@ final class TransportHandshaker { static final TransportVersion EARLIEST_HANDSHAKE_VERSION = TransportVersion.fromId(6080099); static final TransportVersion REQUEST_HANDSHAKE_VERSION = TransportVersions.MINIMUM_COMPATIBLE; - static final Set ALLOWED_HANDSHAKE_VERSIONS = Set.of(EARLIEST_HANDSHAKE_VERSION, REQUEST_HANDSHAKE_VERSION); + static final TransportVersion V9_HANDSHAKE_VERSION = TransportVersion.fromId(8_800_00_0); + static final Set ALLOWED_HANDSHAKE_VERSIONS = Set.of( + EARLIEST_HANDSHAKE_VERSION, + REQUEST_HANDSHAKE_VERSION, + V9_HANDSHAKE_VERSION + ); static final String HANDSHAKE_ACTION_NAME = "internal:tcp/handshake"; private final ConcurrentMap pendingHandshakes = new ConcurrentHashMap<>(); diff --git a/server/src/test/java/org/elasticsearch/transport/InboundDecoderTests.java b/server/src/test/java/org/elasticsearch/transport/InboundDecoderTests.java index 889afab00e830..c0185832d6122 100644 --- a/server/src/test/java/org/elasticsearch/transport/InboundDecoderTests.java +++ b/server/src/test/java/org/elasticsearch/transport/InboundDecoderTests.java @@ -27,6 +27,7 @@ import static org.elasticsearch.common.bytes.ReleasableBytesReferenceStreamInputTests.wrapAsReleasable; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasItems; import static org.hamcrest.Matchers.instanceOf; @@ -182,7 +183,7 @@ public void testDecodePreHeaderSizeVariableInt() throws IOException { } } - public void testDecodeHandshakeCompatibility() throws IOException { + public void testDecodeHandshakeV7Compatibility() throws IOException { String action = "test-request"; long requestId = randomNonNegativeLong(); final String headerKey = randomAlphaOfLength(10); @@ -223,6 +224,55 @@ public void testDecodeHandshakeCompatibility() throws IOException { } + public void testDecodeHandshakeV8Compatibility() throws IOException { + doHandshakeCompatibilityTest(TransportHandshaker.REQUEST_HANDSHAKE_VERSION, null); + doHandshakeCompatibilityTest(TransportHandshaker.REQUEST_HANDSHAKE_VERSION, Compression.Scheme.DEFLATE); + } + + public void testDecodeHandshakeV9Compatibility() throws IOException { + doHandshakeCompatibilityTest(TransportHandshaker.V9_HANDSHAKE_VERSION, null); + doHandshakeCompatibilityTest(TransportHandshaker.V9_HANDSHAKE_VERSION, Compression.Scheme.DEFLATE); + } + + private void doHandshakeCompatibilityTest(TransportVersion transportVersion, Compression.Scheme compressionScheme) throws IOException { + String action = "test-request"; + long requestId = randomNonNegativeLong(); + final String headerKey = randomAlphaOfLength(10); + final String headerValue = randomAlphaOfLength(20); + threadContext.putHeader(headerKey, headerValue); + OutboundMessage message = new OutboundMessage.Request( + threadContext, + new TestRequest(randomAlphaOfLength(100)), + transportVersion, + action, + requestId, + true, + compressionScheme + ); + + try (RecyclerBytesStreamOutput os = new RecyclerBytesStreamOutput(recycler)) { + final BytesReference bytes = message.serialize(os); + int totalHeaderSize = TcpHeader.headerSize(transportVersion); + + InboundDecoder decoder = new InboundDecoder(recycler); + final ArrayList fragments = new ArrayList<>(); + final ReleasableBytesReference releasable1 = wrapAsReleasable(bytes); + int bytesConsumed = decoder.decode(releasable1, fragments::add); + assertThat(bytesConsumed, greaterThan(totalHeaderSize)); + assertTrue(releasable1.hasReferences()); + + final Header header = (Header) fragments.get(0); + assertEquals(requestId, header.getRequestId()); + assertEquals(transportVersion, header.getVersion()); + assertEquals(compressionScheme == Compression.Scheme.DEFLATE, header.isCompressed()); + assertTrue(header.isHandshake()); + assertTrue(header.isRequest()); + assertFalse(header.needsToReadVariableHeader()); + assertEquals(headerValue, header.getRequestHeaders().get(headerKey)); + fragments.clear(); + } + } + public void testClientChannelTypeFailsDecodingRequests() throws Exception { String action = "test-request"; long requestId = randomNonNegativeLong(); @@ -488,23 +538,16 @@ public void testCheckVersionCompatibility() { } public void testCheckHandshakeCompatibility() { - try { - InboundDecoder.checkHandshakeVersionCompatibility(randomFrom(TransportHandshaker.ALLOWED_HANDSHAKE_VERSIONS)); - } catch (IllegalStateException e) { - throw new AssertionError(e); - } + for (final var allowedHandshakeVersion : TransportHandshaker.ALLOWED_HANDSHAKE_VERSIONS) { + InboundDecoder.checkHandshakeVersionCompatibility(allowedHandshakeVersion); // should not throw - var invalid = TransportVersion.fromId(TransportHandshaker.EARLIEST_HANDSHAKE_VERSION.id() - 1); - try { - InboundDecoder.checkHandshakeVersionCompatibility(invalid); - fail(); - } catch (IllegalStateException expected) { + var invalid = TransportVersion.fromId(allowedHandshakeVersion.id() + randomFrom(-1, +1)); assertEquals( "Received message from unsupported version: [" + invalid + "] allowed versions are: " + TransportHandshaker.ALLOWED_HANDSHAKE_VERSIONS, - expected.getMessage() + expectThrows(IllegalStateException.class, () -> InboundDecoder.checkHandshakeVersionCompatibility(invalid)).getMessage() ); } } From cd82da515e0cdd392bf72fa19e8cdc293b5079ae Mon Sep 17 00:00:00 2001 From: Parker Timmins Date: Tue, 14 Jan 2025 10:11:20 -0600 Subject: [PATCH 28/31] Improve how reindex data stream index action handles api blocks (#120084) - fail quickly if index has read or metadata block - add write block, but proceed if already has read-only or read-only-allow-delete - filter write, read-only, and read-only-allow-delete from dest index - results in dest index having no blocks --- docs/changelog/120084.yaml | 5 ++ ...indexDatastreamIndexTransportActionIT.java | 60 ++++++++++++++----- ...ReindexDataStreamIndexTransportAction.java | 40 +++++++++---- 3 files changed, 80 insertions(+), 25 deletions(-) create mode 100644 docs/changelog/120084.yaml diff --git a/docs/changelog/120084.yaml b/docs/changelog/120084.yaml new file mode 100644 index 0000000000000..aafe490d79f1e --- /dev/null +++ b/docs/changelog/120084.yaml @@ -0,0 +1,5 @@ +pr: 120084 +summary: Improve how reindex data stream index action handles api blocks +area: Data streams +type: enhancement +issues: [] diff --git a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java index 71b2a2a43b343..b310d332fdd5c 100644 --- a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java +++ b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.migrate.action; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; @@ -224,16 +225,46 @@ public void testMappingsAddedToDestIndex() throws Exception { assertEquals("text", XContentMapValues.extractValue("properties.foo1.type", destMappings)); } - public void testReadOnlyAddedBack() { + public void testFailIfMetadataBlockSet() { + assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled()); + + var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + var settings = Settings.builder().put(IndexMetadata.SETTING_BLOCKS_METADATA, true).build(); + indicesAdmin().create(new CreateIndexRequest(sourceIndex, settings)).actionGet(); + + try { + client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)).actionGet(); + } catch (ElasticsearchException e) { + assertTrue(e.getMessage().contains("Cannot reindex index") || e.getCause().getMessage().equals("Cannot reindex index")); + } + + cleanupMetadataBlocks(sourceIndex); + } + + public void testFailIfReadBlockSet() { + assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled()); + + var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + var settings = Settings.builder().put(IndexMetadata.SETTING_BLOCKS_READ, true).build(); + indicesAdmin().create(new CreateIndexRequest(sourceIndex, settings)).actionGet(); + + try { + client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)).actionGet(); + } catch (ElasticsearchException e) { + assertTrue(e.getMessage().contains("Cannot reindex index") || e.getCause().getMessage().equals("Cannot reindex index")); + } + + cleanupMetadataBlocks(sourceIndex); + } + + public void testReadOnlyBlocksNotAddedBack() { assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled()); - // Create source index with read-only and/or block-writes var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); - boolean isReadOnly = randomBoolean(); - boolean isBlockWrites = randomBoolean(); var settings = Settings.builder() - .put(IndexMetadata.SETTING_READ_ONLY, isReadOnly) - .put(IndexMetadata.SETTING_BLOCKS_WRITE, isBlockWrites) + .put(IndexMetadata.SETTING_READ_ONLY, randomBoolean()) + .put(IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE, randomBoolean()) + .put(IndexMetadata.SETTING_BLOCKS_WRITE, randomBoolean()) .build(); indicesAdmin().create(new CreateIndexRequest(sourceIndex, settings)).actionGet(); @@ -242,13 +273,13 @@ public void testReadOnlyAddedBack() { .actionGet() .getDestIndex(); - // assert read-only settings added back to dest index var settingsResponse = indicesAdmin().getSettings(new GetSettingsRequest().indices(destIndex)).actionGet(); - assertEquals(isReadOnly, Boolean.parseBoolean(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_READ_ONLY))); - assertEquals(isBlockWrites, Boolean.parseBoolean(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_BLOCKS_WRITE))); + assertFalse(Boolean.parseBoolean(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_READ_ONLY))); + assertFalse(Boolean.parseBoolean(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE))); + assertFalse(Boolean.parseBoolean(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_BLOCKS_WRITE))); - removeReadOnly(sourceIndex); - removeReadOnly(destIndex); + cleanupMetadataBlocks(sourceIndex); + cleanupMetadataBlocks(destIndex); } public void testUpdateSettingsDefaultsRestored() { @@ -428,10 +459,11 @@ public void testTsdbStartEndSet() throws Exception { // TODO check other IndexMetadata fields that need to be fixed after the fact // TODO what happens if don't have necessary perms for a given index? - private static void removeReadOnly(String index) { + private static void cleanupMetadataBlocks(String index) { var settings = Settings.builder() - .put(IndexMetadata.SETTING_READ_ONLY, false) - .put(IndexMetadata.SETTING_BLOCKS_WRITE, false) + .putNull(IndexMetadata.SETTING_READ_ONLY) + .putNull(IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE) + .putNull(IndexMetadata.SETTING_BLOCKS_METADATA) .build(); assertAcked(indicesAdmin().updateSettings(new UpdateSettingsRequest(settings, index)).actionGet()); } diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java index ff350429dae01..7bb440bc52a15 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java @@ -40,7 +40,6 @@ import java.util.Locale; import java.util.Map; -import static org.elasticsearch.cluster.metadata.IndexMetadata.APIBlock.READ_ONLY; import static org.elasticsearch.cluster.metadata.IndexMetadata.APIBlock.WRITE; public class ReindexDataStreamIndexTransportAction extends HandledTransportAction< @@ -93,13 +92,22 @@ protected void doExecute( ); } + if (settingsBefore.getAsBoolean(IndexMetadata.SETTING_BLOCKS_READ, false)) { + var errorMessage = String.format(Locale.ROOT, "Cannot reindex index [%s] which has a read block.", destIndexName); + listener.onFailure(new ElasticsearchException(errorMessage)); + return; + } + if (settingsBefore.getAsBoolean(IndexMetadata.SETTING_BLOCKS_METADATA, false)) { + var errorMessage = String.format(Locale.ROOT, "Cannot reindex index [%s] which has a metadata block.", destIndexName); + listener.onFailure(new ElasticsearchException(errorMessage)); + return; + } + SubscribableListener.newForked(l -> setBlockWrites(sourceIndexName, l, taskId)) .andThen(l -> deleteDestIfExists(destIndexName, l, taskId)) .andThen(l -> createIndex(sourceIndex, destIndexName, l, taskId)) .andThen(l -> reindex(sourceIndexName, destIndexName, l, taskId)) .andThen(l -> copyOldSourceSettingsToDest(settingsBefore, destIndexName, l, taskId)) - .andThen(l -> addBlockIfFromSource(WRITE, settingsBefore, destIndexName, l, taskId)) - .andThen(l -> addBlockIfFromSource(READ_ONLY, settingsBefore, destIndexName, l, taskId)) .andThenApply(ignored -> new ReindexDataStreamIndexAction.Response(destIndexName)) .addListener(listener); } @@ -120,7 +128,8 @@ public void onResponse(AddIndexBlockResponse response) { @Override public void onFailure(Exception e) { if (e instanceof ClusterBlockException || e.getCause() instanceof ClusterBlockException) { - // It's fine if block-writes is already set + // Could fail with a cluster block exception if read-only or read-only-allow-delete is already set + // In this case, we can proceed listener.onResponse(null); } else { listener.onFailure(e); @@ -146,10 +155,12 @@ private void createIndex( ) { logger.debug("Creating destination index [{}] for source index [{}]", destIndexName, sourceIndex.getIndex().getName()); - // override read-only settings if they exist var removeReadOnlyOverride = Settings.builder() + // remove read-only settings if they exist .putNull(IndexMetadata.SETTING_READ_ONLY) + .putNull(IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE) .putNull(IndexMetadata.SETTING_BLOCKS_WRITE) + // settings to optimize reindex .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), -1) .build(); @@ -192,6 +203,18 @@ private void addBlockIfFromSource( } } + private void updateSettings( + String index, + Settings.Builder settings, + ActionListener listener, + TaskId parentTaskId + ) { + var updateSettingsRequest = new UpdateSettingsRequest(settings.build(), index); + updateSettingsRequest.setParentTask(parentTaskId); + var errorMessage = String.format(Locale.ROOT, "Could not update settings on index [%s]", index); + client.admin().indices().updateSettings(updateSettingsRequest, failIfNotAcknowledged(listener, errorMessage)); + } + private void copyOldSourceSettingsToDest( Settings settingsBefore, String destIndexName, @@ -199,15 +222,10 @@ private void copyOldSourceSettingsToDest( TaskId parentTaskId ) { logger.debug("Updating settings on destination index after reindex completes"); - var settings = Settings.builder(); copySettingOrUnset(settingsBefore, settings, IndexMetadata.SETTING_NUMBER_OF_REPLICAS); copySettingOrUnset(settingsBefore, settings, IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey()); - - var updateSettingsRequest = new UpdateSettingsRequest(settings.build(), destIndexName); - updateSettingsRequest.setParentTask(parentTaskId); - var errorMessage = String.format(Locale.ROOT, "Could not update settings on index [%s]", destIndexName); - client.admin().indices().updateSettings(updateSettingsRequest, failIfNotAcknowledged(listener, errorMessage)); + updateSettings(destIndexName, settings, listener, parentTaskId); } private static void copySettingOrUnset(Settings settingsBefore, Settings.Builder builder, String setting) { From e196138b3cf90d7b11c5ae2f6afc16761e5ab33c Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Tue, 14 Jan 2025 17:23:12 +0100 Subject: [PATCH 29/31] [CI] Unmute RollingUpgradeSearchableSnapshotIndexCompatibilityIT tests (#120113) Mixed cluster tests failed because a serialization change wasn't backported during a couple of days (#119832). Now the change is backported, we can reenable those tests. Closes #119990 Closes #119989 Closes #119980 Closes #119979 Closes #119978 --- muted-tests.yml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 5992c3dda800a..4ddc79a51ec7f 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -227,26 +227,8 @@ tests: - class: org.elasticsearch.xpack.inference.InferenceCrudIT method: testGetServicesWithCompletionTaskType issue: https://github.com/elastic/elasticsearch/issues/119959 -- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT - method: testSearchableSnapshotUpgrade {p0=[9.0.0, 8.18.0, 8.18.0]} - issue: https://github.com/elastic/elasticsearch/issues/119978 -- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT - method: testSearchableSnapshotUpgrade {p0=[9.0.0, 9.0.0, 8.18.0]} - issue: https://github.com/elastic/elasticsearch/issues/119979 -- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT - method: testMountSearchableSnapshot {p0=[9.0.0, 8.18.0, 8.18.0]} - issue: https://github.com/elastic/elasticsearch/issues/119550 -- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT - method: testMountSearchableSnapshot {p0=[9.0.0, 9.0.0, 8.18.0]} - issue: https://github.com/elastic/elasticsearch/issues/119980 - class: org.elasticsearch.multi_cluster.MultiClusterYamlTestSuiteIT issue: https://github.com/elastic/elasticsearch/issues/119983 -- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT - method: testMountSearchableSnapshot {p0=[9.0.0, 9.0.0, 9.0.0]} - issue: https://github.com/elastic/elasticsearch/issues/119989 -- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT - method: testSearchableSnapshotUpgrade {p0=[9.0.0, 9.0.0, 9.0.0]} - issue: https://github.com/elastic/elasticsearch/issues/119990 - class: org.elasticsearch.xpack.test.rest.XPackRestIT method: test {p0=transform/transforms_unattended/Test unattended put and start} issue: https://github.com/elastic/elasticsearch/issues/120019 From f9b90075efc3bb3cda1254934986b559df2665ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20R=C3=BChsen?= Date: Tue, 14 Jan 2025 17:38:58 +0100 Subject: [PATCH 30/31] [Profiling] Drop deprecated ChunkedToXContentObject.toString() (#120043) * [Profiling] Drop deprecated ChunkedToXContentObject.toString() * [CI] Auto commit changes from spotless --------- Co-authored-by: elasticsearchmachine --- .../xpack/profiling/action/GetStackTracesResponse.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetStackTracesResponse.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetStackTracesResponse.java index c2dd51acfd76b..6ec1f9a7aa7e1 100644 --- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetStackTracesResponse.java +++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetStackTracesResponse.java @@ -8,7 +8,6 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.support.TransportAction; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; @@ -142,9 +141,4 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(stackTraces, stackFrames, executables, stackTraceEvents, totalFrames, samplingRate); } - - @Override - public String toString() { - return Strings.toString(this, true, true); - } } From 77392c31a85febbd41170c9d904a402f8aab6d23 Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Tue, 14 Jan 2025 20:43:03 +0000 Subject: [PATCH 31/31] Use approximation to advance matched queries (#120133) This PR resolves a regression introduced in #94564 by ensuring that the approximation is used when advancing matched query clauses. Utilizing the two-phase iterator to validate matches guarantees that we do not attempt to find the next document fulfilling the two-phase criteria beyond the current document. This fix prevents scenarios where matching a document in the second phase significantly increases query complexity, especially in cases involving restrictive second-pass filters. Closes #120130 --- docs/changelog/120133.yaml | 6 +++++ .../fetch/subphase/MatchedQueriesPhase.java | 26 +++++++++++++------ 2 files changed, 24 insertions(+), 8 deletions(-) create mode 100644 docs/changelog/120133.yaml diff --git a/docs/changelog/120133.yaml b/docs/changelog/120133.yaml new file mode 100644 index 0000000000000..4ec88267a1bf8 --- /dev/null +++ b/docs/changelog/120133.yaml @@ -0,0 +1,6 @@ +pr: 120133 +summary: Use approximation to advance matched queries +area: Search +type: bug +issues: + - 120130 diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesPhase.java index 5220dadec7a1a..53116ff8f6ebf 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesPhase.java @@ -9,10 +9,12 @@ package org.elasticsearch.search.fetch.subphase; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.ScorerSupplier; +import org.apache.lucene.search.TwoPhaseIterator; import org.apache.lucene.search.Weight; import org.elasticsearch.index.query.ParsedQuery; import org.elasticsearch.search.fetch.FetchContext; @@ -52,8 +54,9 @@ public FetchSubPhaseProcessor getProcessor(FetchContext context) throws IOExcept ); } return new FetchSubPhaseProcessor() { + record ScorerAndIterator(Scorer scorer, DocIdSetIterator approximation, TwoPhaseIterator twoPhase) {} - final Map matchingIterators = new HashMap<>(); + final Map matchingIterators = new HashMap<>(); @Override public void setNextReader(LeafReaderContext readerContext) throws IOException { @@ -63,7 +66,14 @@ public void setNextReader(LeafReaderContext readerContext) throws IOException { if (ss != null) { Scorer scorer = ss.get(0L); if (scorer != null) { - matchingIterators.put(entry.getKey(), scorer); + final TwoPhaseIterator twoPhase = scorer.twoPhaseIterator(); + final DocIdSetIterator iterator; + if (twoPhase == null) { + iterator = scorer.iterator(); + } else { + iterator = twoPhase.approximation(); + } + matchingIterators.put(entry.getKey(), new ScorerAndIterator(scorer, iterator, twoPhase)); } } } @@ -73,13 +83,13 @@ public void setNextReader(LeafReaderContext readerContext) throws IOException { public void process(HitContext hitContext) throws IOException { Map matches = new LinkedHashMap<>(); int doc = hitContext.docId(); - for (Map.Entry entry : matchingIterators.entrySet()) { - Scorer scorer = entry.getValue(); - if (scorer.iterator().docID() < doc) { - scorer.iterator().advance(doc); + for (Map.Entry entry : matchingIterators.entrySet()) { + ScorerAndIterator query = entry.getValue(); + if (query.approximation.docID() < doc) { + query.approximation.advance(doc); } - if (scorer.iterator().docID() == doc) { - matches.put(entry.getKey(), scorer.score()); + if (query.approximation.docID() == doc && (query.twoPhase == null || query.twoPhase.matches())) { + matches.put(entry.getKey(), query.scorer.score()); } } hitContext.hit().matchedQueries(matches);