From 44eecfe68696a555f7d033eedf42a97861bba1aa Mon Sep 17 00:00:00 2001 From: Pat Whelan Date: Wed, 15 Jan 2025 13:52:13 -0500 Subject: [PATCH 1/4] [Transform] add support for extended_stats Building off of `stats` and multi-value aggregations, including the limitation: - all values of extended_stats will be mapped to `double` if mapping deduction is used Relates #51925 --- docs/reference/rest-api/common-parms.asciidoc | 1 + .../ExtendedStatsAggregationBuilder.java | 6 ++ .../integration/TransformPivotRestIT.java | 68 +++++++++++++++++++ .../pivot/TransformAggregations.java | 4 +- .../pivot/TransformAggregationsTests.java | 37 ++++++++++ 5 files changed, 114 insertions(+), 2 deletions(-) diff --git a/docs/reference/rest-api/common-parms.asciidoc b/docs/reference/rest-api/common-parms.asciidoc index 83c11c9256a67..920de7a54ad74 100644 --- a/docs/reference/rest-api/common-parms.asciidoc +++ b/docs/reference/rest-api/common-parms.asciidoc @@ -808,6 +808,7 @@ currently supported: * <> * <> * <> +* <> * <> * <> * <> diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregationBuilder.java index 21bfffa883f50..f612d00262354 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregationBuilder.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; public class ExtendedStatsAggregationBuilder extends ValuesSourceAggregationBuilder.MetricsAggregationBuilder< @@ -87,6 +88,11 @@ public Set metricNames() { return InternalExtendedStats.METRIC_NAMES; } + @Override + public Optional> getOutputFieldNames() { + return Optional.of(InternalExtendedStats.METRIC_NAMES); + } + @Override protected ValuesSourceType defaultValueSourceType() { return CoreValuesSourceType.NUMERIC; diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java index 322ac63a819fc..8c79ee1afb287 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java @@ -2003,6 +2003,74 @@ public void testPivotWithTopMetrics() throws Exception { assertEquals("business_3", actual); } + @SuppressWarnings(value = "unchecked") + public void testPivotWithExtendedMetrics() throws Exception { + String transformId = "extended_metrics_transform"; + String transformIndex = "extended_metrics_pivot_reviews"; + setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, transformIndex); + + final Request createTransformRequest = createRequestWithAuth( + "PUT", + getTransformEndpoint() + transformId, + BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS + ); + + String config = Strings.format(""" + { + "source": { + "index": "%s" + }, + "dest": { + "index": "%s" + }, + "pivot": { + "group_by": { + "reviewer": { + "terms": { + "field": "user_id" + } + } + }, + "aggregations": { + "stars": { + "extended_stats": { + "field": "stars" + } + } + } + } + }""", REVIEWS_INDEX_NAME, transformIndex); + + createTransformRequest.setJsonEntity(config); + Map createTransformResponse = entityAsMap(client().performRequest(createTransformRequest)); + assertThat(createTransformResponse.get("acknowledged"), equalTo(Boolean.TRUE)); + + startAndWaitForTransform(transformId, transformIndex, BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS); + assertTrue(indexExists(transformIndex)); + + Map searchResult = getAsMap(transformIndex + "/_search?q=reviewer:user_4"); + assertEquals(1, XContentMapValues.extractValue("hits.total.value", searchResult)); + var stdDevMap = (Map) ((List) XContentMapValues.extractValue("hits.hits._source.stars", searchResult)).get(0); + assertThat(stdDevMap.get("count"), is(equalTo(41.0))); + assertThat(stdDevMap.get("sum"), is(equalTo(159.0))); + assertThat(stdDevMap.get("min"), is(equalTo(1.0))); + assertThat(stdDevMap.get("max"), is(equalTo(5.0))); + assertThat(stdDevMap.get("avg"), is(equalTo(3.8780487804878048))); + assertThat(stdDevMap.get("sum_of_squares"), is(equalTo(711.0))); + assertThat(stdDevMap.get("variance"), is(equalTo(2.3022010707911953))); + assertThat(stdDevMap.get("variance_population"), is(equalTo(2.3022010707911953))); + assertThat(stdDevMap.get("variance_sampling"), is(equalTo(2.3597560975609753))); + assertThat(stdDevMap.get("std_deviation"), is(equalTo(1.5173005868288574))); + assertThat(stdDevMap.get("std_deviation_population"), is(equalTo(1.5173005868288574))); + assertThat(stdDevMap.get("std_deviation_sampling"), is(equalTo(1.5361497640402693))); + assertThat(stdDevMap.get("std_upper"), is(equalTo(6.91264995414552))); + assertThat(stdDevMap.get("std_lower"), is(equalTo(0.84344760683009))); + assertThat(stdDevMap.get("std_upper_population"), is(equalTo(6.91264995414552))); + assertThat(stdDevMap.get("std_lower_population"), is(equalTo(0.84344760683009))); + assertThat(stdDevMap.get("std_upper_sampling"), is(equalTo(6.950348308568343))); + assertThat(stdDevMap.get("std_lower_sampling"), is(equalTo(0.8057492524072662))); + } + public void testPivotWithBoxplot() throws Exception { String transformId = "boxplot_transform"; String transformIndex = "boxplot_pivot_reviews"; diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/TransformAggregations.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/TransformAggregations.java index 95e05d93ff03a..16ad1eb8fcd51 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/TransformAggregations.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/TransformAggregations.java @@ -60,7 +60,6 @@ public final class TransformAggregations { "date_histogram", "date_range", "diversified_sampler", - "extended_stats", // https://github.com/elastic/elasticsearch/issues/51925 "filters", "geo_distance", "geohash_grid", @@ -120,7 +119,8 @@ enum AggregationType { MISSING("missing", LONG), TOP_METRICS("top_metrics", SOURCE), STATS("stats", DOUBLE), - BOXPLOT("boxplot", DOUBLE); + BOXPLOT("boxplot", DOUBLE), + EXTENDED_STATS("extended_stats", DOUBLE); private final String aggregationType; private final String targetMapping; diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/TransformAggregationsTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/TransformAggregationsTests.java index 4564ec5cc67ea..f1aa0b9e17b00 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/TransformAggregationsTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/TransformAggregationsTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.ExtendedStatsAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.MinAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.PercentilesAggregationBuilder; @@ -31,7 +32,9 @@ import java.util.Map; import java.util.stream.Collectors; +import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.is; public class TransformAggregationsTests extends ESTestCase { @@ -137,6 +140,9 @@ public void testResolveTargetMapping() { assertEquals("double", TransformAggregations.resolveTargetMapping("stats", null)); assertEquals("double", TransformAggregations.resolveTargetMapping("stats", "int")); + // extended stats + assertEquals("double", TransformAggregations.resolveTargetMapping("extended_stats", "double")); + // boxplot assertEquals("double", TransformAggregations.resolveTargetMapping("boxplot", "double")); @@ -220,6 +226,37 @@ public void testGetAggregationOutputTypesStats() { assertEquals("stats", outputTypes.get("stats.sum")); } + public void testGetAggregationOutputTypesExtendedStats() { + var extendedStatsAggregationBuilder = new ExtendedStatsAggregationBuilder("extended_stats"); + + var inputAndOutputTypes = TransformAggregations.getAggregationInputAndOutputTypes(extendedStatsAggregationBuilder); + var outputTypes = inputAndOutputTypes.v2(); + assertEquals(18, outputTypes.size()); + assertThat( + outputTypes, + allOf( + hasEntry("extended_stats.std_upper_population", "extended_stats"), + hasEntry("extended_stats.variance", "extended_stats"), + hasEntry("extended_stats.avg", "extended_stats"), + hasEntry("extended_stats.min", "extended_stats"), + hasEntry("extended_stats.sum_of_squares", "extended_stats"), + hasEntry("extended_stats.sum", "extended_stats"), + hasEntry("extended_stats.variance_sampling", "extended_stats"), + hasEntry("extended_stats.std_lower_population", "extended_stats"), + hasEntry("extended_stats.std_deviation", "extended_stats"), + hasEntry("extended_stats.std_upper_sampling", "extended_stats"), + hasEntry("extended_stats.std_deviation_population", "extended_stats"), + hasEntry("extended_stats.max", "extended_stats"), + hasEntry("extended_stats.std_lower_sampling", "extended_stats"), + hasEntry("extended_stats.std_deviation_sampling", "extended_stats"), + hasEntry("extended_stats.std_upper", "extended_stats"), + hasEntry("extended_stats.count", "extended_stats"), + hasEntry("extended_stats.variance_population", "extended_stats"), + hasEntry("extended_stats.std_lower", "extended_stats") + ) + ); + } + public void testGetAggregationOutputTypesRange() { { AggregationBuilder rangeAggregationBuilder = new RangeAggregationBuilder("range_agg_name").addUnboundedTo(100) From 8472352d40aaaf253356bb331e8b037842911b75 Mon Sep 17 00:00:00 2001 From: Pat Whelan Date: Thu, 16 Jan 2025 19:34:36 -0500 Subject: [PATCH 2/4] Update docs/changelog/120340.yaml --- docs/changelog/120340.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/120340.yaml diff --git a/docs/changelog/120340.yaml b/docs/changelog/120340.yaml new file mode 100644 index 0000000000000..3c2200170c0c4 --- /dev/null +++ b/docs/changelog/120340.yaml @@ -0,0 +1,5 @@ +pr: 120340 +summary: Add support for `extended_stats` +area: Transform +type: enhancement +issues: [] From b24b2f3ff47294e16a5c3c7299e5046c91be64b2 Mon Sep 17 00:00:00 2001 From: Pat Whelan Date: Tue, 21 Jan 2025 10:24:45 -0500 Subject: [PATCH 3/4] Mimic xcontent structure --- .../ExtendedStatsAggregationBuilder.java | 2 +- .../metrics/InternalExtendedStats.java | 63 ++++++++++++++ .../metrics/InternalExtendedStatsTests.java | 84 +++++++++++++++++++ .../integration/TransformPivotRestIT.java | 60 +++++++------ .../pivot/AggregationResultUtils.java | 12 +++ .../pivot/TransformAggregationsTests.java | 24 +++--- 6 files changed, 208 insertions(+), 37 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregationBuilder.java index f612d00262354..af47141730e60 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregationBuilder.java @@ -90,7 +90,7 @@ public Set metricNames() { @Override public Optional> getOutputFieldNames() { - return Optional.of(InternalExtendedStats.METRIC_NAMES); + return Optional.of(InternalExtendedStats.Fields.OUTPUT_FORMAT); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalExtendedStats.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalExtendedStats.java index f74206c7af8b7..7965211e24683 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalExtendedStats.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalExtendedStats.java @@ -8,6 +8,7 @@ */ package org.elasticsearch.search.aggregations.metrics; +import org.elasticsearch.common.TriConsumer; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.search.DocValueFormat; @@ -19,6 +20,7 @@ import java.io.IOException; import java.util.Collections; +import java.util.HashMap; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -337,6 +339,67 @@ static class Fields { public static final String LOWER_POPULATION = "lower_population"; public static final String UPPER_SAMPLING = "upper_sampling"; public static final String LOWER_SAMPLING = "lower_sampling"; + + static final Set OUTPUT_FORMAT = Set.of( + Metrics.count.name(), + Metrics.sum.name(), + Metrics.min.name(), + Metrics.max.name(), + Metrics.avg.name(), + SUM_OF_SQRS, + VARIANCE, + VARIANCE_POPULATION, + VARIANCE_SAMPLING, + STD_DEVIATION, + STD_DEVIATION_POPULATION, + STD_DEVIATION_SAMPLING, + STD_DEVIATION_BOUNDS + "." + UPPER, + STD_DEVIATION_BOUNDS + "." + LOWER, + STD_DEVIATION_BOUNDS + "." + UPPER_POPULATION, + STD_DEVIATION_BOUNDS + "." + LOWER_POPULATION, + STD_DEVIATION_BOUNDS + "." + UPPER_SAMPLING, + STD_DEVIATION_BOUNDS + "." + LOWER_SAMPLING + ); + } + + public Map asIndexableMap() { + if (count != 0) { + // NumberFieldMapper will invalidate non-finite doubles + TriConsumer, String, Double> putIfValidDouble = (map, key, value) -> { + if (Double.isFinite(value)) { + map.put(key, value); + } + }; + var extendedStatsMap = new HashMap(13); + extendedStatsMap.put(Metrics.count.name(), getCount()); + putIfValidDouble.apply(extendedStatsMap, Metrics.sum.name(), getSum()); + putIfValidDouble.apply(extendedStatsMap, Metrics.min.name(), getMin()); + putIfValidDouble.apply(extendedStatsMap, Metrics.max.name(), getMax()); + putIfValidDouble.apply(extendedStatsMap, Metrics.avg.name(), getAvg()); + + putIfValidDouble.apply(extendedStatsMap, Fields.SUM_OF_SQRS, sumOfSqrs); + putIfValidDouble.apply(extendedStatsMap, Fields.VARIANCE, getVariance()); + putIfValidDouble.apply(extendedStatsMap, Fields.VARIANCE_POPULATION, getVariancePopulation()); + putIfValidDouble.apply(extendedStatsMap, Fields.VARIANCE_SAMPLING, getVarianceSampling()); + putIfValidDouble.apply(extendedStatsMap, Fields.STD_DEVIATION, getStdDeviation()); + putIfValidDouble.apply(extendedStatsMap, Fields.STD_DEVIATION_POPULATION, getStdDeviationPopulation()); + putIfValidDouble.apply(extendedStatsMap, Fields.STD_DEVIATION_SAMPLING, getStdDeviationSampling()); + + var stdDevBounds = new HashMap(6); + putIfValidDouble.apply(stdDevBounds, Fields.UPPER, getStdDeviationBound(Bounds.UPPER)); + putIfValidDouble.apply(stdDevBounds, Fields.LOWER, getStdDeviationBound(Bounds.LOWER)); + putIfValidDouble.apply(stdDevBounds, Fields.UPPER_POPULATION, getStdDeviationBound(Bounds.UPPER_POPULATION)); + putIfValidDouble.apply(stdDevBounds, Fields.LOWER_POPULATION, getStdDeviationBound(Bounds.LOWER_POPULATION)); + putIfValidDouble.apply(stdDevBounds, Fields.UPPER_SAMPLING, getStdDeviationBound(Bounds.UPPER_SAMPLING)); + putIfValidDouble.apply(stdDevBounds, Fields.LOWER_SAMPLING, getStdDeviationBound(Bounds.LOWER_SAMPLING)); + if (stdDevBounds.isEmpty() == false) { + extendedStatsMap.put(Fields.STD_DEVIATION_BOUNDS, stdDevBounds); + } + + return extendedStatsMap; + } else { + return Map.of(); + } } @Override diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalExtendedStatsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalExtendedStatsTests.java index bbb007c9155ba..b919428c00ef9 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalExtendedStatsTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalExtendedStatsTests.java @@ -9,16 +9,29 @@ package org.elasticsearch.search.aggregations.metrics; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.util.Maps; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.support.SamplingContext; import org.elasticsearch.test.InternalAggregationTestCase; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentFactory; +import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Predicate; + +import static org.hamcrest.Matchers.aMapWithSize; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.isA; +import static org.hamcrest.Matchers.notNullValue; public class InternalExtendedStatsTests extends InternalAggregationTestCase { @@ -209,4 +222,75 @@ private void verifySumOfSqrsOfDoubles(double[] values, double expectedSumOfSqrs, InternalExtendedStats reduced = (InternalExtendedStats) InternalAggregationTestCase.reduce(aggregations, null); assertEquals(expectedSumOfSqrs, reduced.getSumOfSquares(), delta); } + + @SuppressWarnings(value = "unchecked") + public void testAsMapMatchesXContent() throws IOException { + var stats = new InternalExtendedStats( + "testAsMapIsSameAsXContent", + randomLongBetween(1, 50), + randomDoubleBetween(1, 50, true), + randomDoubleBetween(1, 50, true), + randomDoubleBetween(1, 50, true), + randomDoubleBetween(1, 50, true), + sigma, + DocValueFormat.RAW, + Map.of() + ); + + var outputMap = stats.asIndexableMap(); + assertThat(outputMap, notNullValue()); + + Map xContentMap; + try (var builder = XContentFactory.jsonBuilder()) { + builder.startObject(); + stats.doXContentBody(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + xContentMap = XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2(); + } + assertThat(xContentMap, notNullValue()); + + // serializing -> deserializing converts the long to an int, so we convert it back to test + var countMetricName = InternalStats.Metrics.count.name(); + var xContentCount = xContentMap.get(countMetricName); + assertThat(xContentCount, isA(Integer.class)); + assertThat(((Integer) xContentCount).longValue(), equalTo(outputMap.get(countMetricName))); + + // verify the entries in the bounds map are similar + var xContentStdDevBounds = (Map) xContentMap.get(InternalExtendedStats.Fields.STD_DEVIATION_BOUNDS); + var outputStdDevBounds = (Map) outputMap.get(InternalExtendedStats.Fields.STD_DEVIATION_BOUNDS); + xContentStdDevBounds.forEach((key, value) -> { + if (value instanceof String == false || Double.isFinite(Double.parseDouble(value.toString()))) { + assertThat(outputStdDevBounds.get(key), equalTo(value)); + } + }); + + // verify all the other entries that are not "std_deviation_bounds" or "count" + Predicate> notCountOrStdDevBounds = Predicate.not( + e -> e.getKey().equals(countMetricName) || e.getKey().equals(InternalExtendedStats.Fields.STD_DEVIATION_BOUNDS) + ); + xContentMap.entrySet().stream().filter(notCountOrStdDevBounds).forEach(e -> { + if (e.getValue() instanceof String == false || Double.isFinite(Double.parseDouble(e.getValue().toString()))) { + assertThat(outputMap.get(e.getKey()), equalTo(e.getValue())); + } + }); + } + + public void testIndexableMapExcludesNaN() { + var stats = new InternalExtendedStats( + "testAsMapIsSameAsXContent", + randomLongBetween(1, 50), + Double.NaN, + Double.NaN, + Double.NaN, + Double.NaN, + sigma, + DocValueFormat.RAW, + Map.of() + ); + + var outputMap = stats.asIndexableMap(); + assertThat(outputMap, is(aMapWithSize(1))); + assertThat(outputMap, hasKey(InternalStats.Metrics.count.name())); + assertThat(outputMap.get(InternalStats.Metrics.count.name()), is(stats.getCount())); + } } diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java index 8c79ee1afb287..83f731e298159 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java @@ -2004,18 +2004,18 @@ public void testPivotWithTopMetrics() throws Exception { } @SuppressWarnings(value = "unchecked") - public void testPivotWithExtendedMetrics() throws Exception { - String transformId = "extended_metrics_transform"; - String transformIndex = "extended_metrics_pivot_reviews"; + public void testPivotWithExtendedStats() throws Exception { + var transformId = "extended_stats_transform"; + var transformIndex = "extended_stats_pivot_reviews"; setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, transformIndex); - final Request createTransformRequest = createRequestWithAuth( + var createTransformRequest = createRequestWithAuth( "PUT", getTransformEndpoint() + transformId, BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS ); - String config = Strings.format(""" + var config = Strings.format(""" { "source": { "index": "%s" @@ -2042,33 +2042,43 @@ public void testPivotWithExtendedMetrics() throws Exception { }""", REVIEWS_INDEX_NAME, transformIndex); createTransformRequest.setJsonEntity(config); - Map createTransformResponse = entityAsMap(client().performRequest(createTransformRequest)); + var createTransformResponse = entityAsMap(client().performRequest(createTransformRequest)); assertThat(createTransformResponse.get("acknowledged"), equalTo(Boolean.TRUE)); startAndWaitForTransform(transformId, transformIndex, BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS); assertTrue(indexExists(transformIndex)); - Map searchResult = getAsMap(transformIndex + "/_search?q=reviewer:user_4"); + var searchResult = getAsMap(transformIndex + "/_search?q=reviewer:user_4"); assertEquals(1, XContentMapValues.extractValue("hits.total.value", searchResult)); var stdDevMap = (Map) ((List) XContentMapValues.extractValue("hits.hits._source.stars", searchResult)).get(0); - assertThat(stdDevMap.get("count"), is(equalTo(41.0))); - assertThat(stdDevMap.get("sum"), is(equalTo(159.0))); - assertThat(stdDevMap.get("min"), is(equalTo(1.0))); - assertThat(stdDevMap.get("max"), is(equalTo(5.0))); - assertThat(stdDevMap.get("avg"), is(equalTo(3.8780487804878048))); - assertThat(stdDevMap.get("sum_of_squares"), is(equalTo(711.0))); - assertThat(stdDevMap.get("variance"), is(equalTo(2.3022010707911953))); - assertThat(stdDevMap.get("variance_population"), is(equalTo(2.3022010707911953))); - assertThat(stdDevMap.get("variance_sampling"), is(equalTo(2.3597560975609753))); - assertThat(stdDevMap.get("std_deviation"), is(equalTo(1.5173005868288574))); - assertThat(stdDevMap.get("std_deviation_population"), is(equalTo(1.5173005868288574))); - assertThat(stdDevMap.get("std_deviation_sampling"), is(equalTo(1.5361497640402693))); - assertThat(stdDevMap.get("std_upper"), is(equalTo(6.91264995414552))); - assertThat(stdDevMap.get("std_lower"), is(equalTo(0.84344760683009))); - assertThat(stdDevMap.get("std_upper_population"), is(equalTo(6.91264995414552))); - assertThat(stdDevMap.get("std_lower_population"), is(equalTo(0.84344760683009))); - assertThat(stdDevMap.get("std_upper_sampling"), is(equalTo(6.950348308568343))); - assertThat(stdDevMap.get("std_lower_sampling"), is(equalTo(0.8057492524072662))); + assertThat(stdDevMap.get("count"), equalTo(41)); + assertThat( + stdDevMap, + allOf( + hasEntry("sum", 159.0), + hasEntry("min", 1.0), + hasEntry("max", 5.0), + hasEntry("avg", 3.8780487804878048), + hasEntry("sum_of_squares", 711.0), + hasEntry("variance", 2.3022010707911953), + hasEntry("variance_population", 2.3022010707911953), + hasEntry("variance_sampling", 2.3597560975609753), + hasEntry("std_deviation", 1.5173005868288574), + hasEntry("std_deviation_sampling", 1.5361497640402693), + hasEntry("std_deviation_population", 1.5173005868288574) + ) + ); + assertThat( + (Map) stdDevMap.get("std_deviation_bounds"), + allOf( + hasEntry("upper", 6.91264995414552), + hasEntry("lower", 0.84344760683009), + hasEntry("upper_population", 6.91264995414552), + hasEntry("lower_population", 0.84344760683009), + hasEntry("upper_sampling", 6.950348308568343), + hasEntry("lower_sampling", 0.8057492524072662) + ) + ); } public void testPivotWithBoxplot() throws Exception { diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtils.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtils.java index a851e4a47f1cc..663b2acb0a01b 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtils.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtils.java @@ -23,6 +23,7 @@ import org.elasticsearch.search.aggregations.bucket.range.Range; import org.elasticsearch.search.aggregations.metrics.GeoBounds; import org.elasticsearch.search.aggregations.metrics.GeoCentroid; +import org.elasticsearch.search.aggregations.metrics.InternalExtendedStats; import org.elasticsearch.search.aggregations.metrics.MultiValueAggregation; import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation.MultiValue; import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation.SingleValue; @@ -69,6 +70,7 @@ public final class AggregationResultUtils { tempMap.put(GeoShapeMetricAggregation.class.getName(), new GeoShapeMetricAggExtractor()); tempMap.put(MultiValue.class.getName(), new NumericMultiValueAggExtractor()); tempMap.put(MultiValueAggregation.class.getName(), new MultiValueAggExtractor()); + tempMap.put(InternalExtendedStats.class.getName(), new ExtendedStatsExtractor()); TYPE_VALUE_EXTRACTOR_MAP = Collections.unmodifiableMap(tempMap); } @@ -171,6 +173,9 @@ static AggValueExtractor getExtractor(Aggregation aggregation) { // TODO: can the Range extractor be removed? } else if (aggregation instanceof Range) { return TYPE_VALUE_EXTRACTOR_MAP.get(Range.class.getName()); + } else if (aggregation instanceof InternalExtendedStats) { + // note: extended stats is also a multi bucket agg, therefore check range first + return TYPE_VALUE_EXTRACTOR_MAP.get(InternalExtendedStats.class.getName()); } else if (aggregation instanceof MultiValue) { return TYPE_VALUE_EXTRACTOR_MAP.get(MultiValue.class.getName()); } else if (aggregation instanceof MultiValueAggregation) { @@ -281,6 +286,13 @@ public Object value(Aggregation agg, Map fieldTypeMap, String lo } } + static class ExtendedStatsExtractor implements AggValueExtractor { + @Override + public Object value(Aggregation agg, Map fieldTypeMap, String lookupFieldPrefix) { + return ((InternalExtendedStats) agg).asIndexableMap(); + } + } + static class MultiValueAggExtractor implements AggValueExtractor { @Override public Object value(Aggregation agg, Map fieldTypeMap, String lookupFieldPrefix) { diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/TransformAggregationsTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/TransformAggregationsTests.java index f1aa0b9e17b00..41a913ec4c2b6 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/TransformAggregationsTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/TransformAggregationsTests.java @@ -235,24 +235,26 @@ public void testGetAggregationOutputTypesExtendedStats() { assertThat( outputTypes, allOf( - hasEntry("extended_stats.std_upper_population", "extended_stats"), - hasEntry("extended_stats.variance", "extended_stats"), + hasEntry("extended_stats.count", "extended_stats"), + hasEntry("extended_stats.sum", "extended_stats"), hasEntry("extended_stats.avg", "extended_stats"), hasEntry("extended_stats.min", "extended_stats"), + hasEntry("extended_stats.max", "extended_stats"), + hasEntry("extended_stats.sum_of_squares", "extended_stats"), - hasEntry("extended_stats.sum", "extended_stats"), + hasEntry("extended_stats.variance", "extended_stats"), + hasEntry("extended_stats.variance_population", "extended_stats"), hasEntry("extended_stats.variance_sampling", "extended_stats"), - hasEntry("extended_stats.std_lower_population", "extended_stats"), hasEntry("extended_stats.std_deviation", "extended_stats"), - hasEntry("extended_stats.std_upper_sampling", "extended_stats"), hasEntry("extended_stats.std_deviation_population", "extended_stats"), - hasEntry("extended_stats.max", "extended_stats"), - hasEntry("extended_stats.std_lower_sampling", "extended_stats"), hasEntry("extended_stats.std_deviation_sampling", "extended_stats"), - hasEntry("extended_stats.std_upper", "extended_stats"), - hasEntry("extended_stats.count", "extended_stats"), - hasEntry("extended_stats.variance_population", "extended_stats"), - hasEntry("extended_stats.std_lower", "extended_stats") + + hasEntry("extended_stats.std_deviation_bounds.upper", "extended_stats"), + hasEntry("extended_stats.std_deviation_bounds.lower", "extended_stats"), + hasEntry("extended_stats.std_deviation_bounds.upper_population", "extended_stats"), + hasEntry("extended_stats.std_deviation_bounds.lower_population", "extended_stats"), + hasEntry("extended_stats.std_deviation_bounds.upper_sampling", "extended_stats"), + hasEntry("extended_stats.std_deviation_bounds.lower_sampling", "extended_stats") ) ); } From 1dbd9364f56affa7b3fb79604c75362f88706a7d Mon Sep 17 00:00:00 2001 From: Pat Whelan Date: Tue, 21 Jan 2025 12:03:05 -0500 Subject: [PATCH 4/4] Fix missing output fields in test --- .../metrics/ExtendedStatsAggregatorTests.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorTests.java index ae4ed3568683a..4151beda6ba0c 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorTests.java @@ -19,7 +19,9 @@ import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorTestCase; +import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; import java.io.IOException; @@ -27,7 +29,7 @@ import java.util.function.Consumer; import static java.util.Collections.singleton; -import static org.elasticsearch.search.aggregations.AggregationBuilders.stats; +import static org.hamcrest.Matchers.is; public class ExtendedStatsAggregatorTests extends AggregatorTestCase { private static final double TOLERANCE = 1e-5; @@ -304,6 +306,13 @@ public void testCase( testCase(buildIndex, verify, new AggTestConfig(aggBuilder, ft)); } + @Override + protected void verifyOutputFieldNames(T aggregationBuilder, V agg) + throws IOException { + assertTrue(aggregationBuilder.getOutputFieldNames().isPresent()); + assertThat(aggregationBuilder.getOutputFieldNames().get(), is(InternalExtendedStats.Fields.OUTPUT_FORMAT)); + } + static class ExtendedSimpleStatsAggregator extends StatsAggregatorTests.SimpleStatsAggregator { double sumOfSqrs = 0;