[Transform] add support for extended_stats

docs/changelog/120340.yaml

@@ -0,0 +1,5 @@
+pr: 120340
+summary: Add support for `extended_stats`
+area: Transform
+type: enhancement
+issues: []

docs/reference/rest-api/common-parms.asciidoc

@@ -808,6 +808,7 @@ currently supported:
 * <<search-aggregations-pipeline-bucket-script-aggregation,Bucket script>>
 * <<search-aggregations-pipeline-bucket-selector-aggregation,Bucket selector>>
 * <<search-aggregations-metrics-cardinality-aggregation,Cardinality>>
+* <<search-aggregations-metrics-extendedstats-aggregation,Extended Stats>>
 * <<search-aggregations-bucket-filter-aggregation,Filter>>
 * <<search-aggregations-metrics-geobounds-aggregation,Geo bounds>>
 * <<search-aggregations-metrics-geocentroid-aggregation,Geo centroid>>

server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregationBuilder.java

@@ -28,6 +28,7 @@
 import java.io.IOException;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 
 public class ExtendedStatsAggregationBuilder extends ValuesSourceAggregationBuilder.MetricsAggregationBuilder<
@@ -87,6 +88,11 @@ public Set<String> metricNames() {
         return InternalExtendedStats.METRIC_NAMES;
     }
 
+    @Override
+    public Optional<Set<String>> getOutputFieldNames() {
+        return Optional.of(InternalExtendedStats.METRIC_NAMES);
+    }
+
     @Override
     protected ValuesSourceType defaultValueSourceType() {
         return CoreValuesSourceType.NUMERIC;

x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java

@@ -2003,6 +2003,74 @@ public void testPivotWithTopMetrics() throws Exception {
         assertEquals("business_3", actual);
     }
 
+    @SuppressWarnings(value = "unchecked")
+    public void testPivotWithExtendedMetrics() throws Exception {
+        String transformId = "extended_metrics_transform";
+        String transformIndex = "extended_metrics_pivot_reviews";
+        setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, transformIndex);
+
+        final Request createTransformRequest = createRequestWithAuth(
+            "PUT",
+            getTransformEndpoint() + transformId,
+            BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS
+        );
+
+        String config = Strings.format("""
+            {
+              "source": {
+                "index": "%s"
+              },
+              "dest": {
+                "index": "%s"
+              },
+              "pivot": {
+                "group_by": {
+                  "reviewer": {
+                    "terms": {
+                      "field": "user_id"
+                    }
+                  }
+                },
+                "aggregations": {
+                  "stars": {
+                    "extended_stats": {
+                      "field": "stars"
+                    }
+                  }
+                }
+              }
+            }""", REVIEWS_INDEX_NAME, transformIndex);
+
+        createTransformRequest.setJsonEntity(config);
+        Map<String, Object> createTransformResponse = entityAsMap(client().performRequest(createTransformRequest));
+        assertThat(createTransformResponse.get("acknowledged"), equalTo(Boolean.TRUE));
+
+        startAndWaitForTransform(transformId, transformIndex, BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS);
+        assertTrue(indexExists(transformIndex));
+
+        Map<String, Object> searchResult = getAsMap(transformIndex + "/_search?q=reviewer:user_4");
+        assertEquals(1, XContentMapValues.extractValue("hits.total.value", searchResult));
+        var stdDevMap = (Map<String, Object>) ((List<?>) XContentMapValues.extractValue("hits.hits._source.stars", searchResult)).get(0);
+        assertThat(stdDevMap.get("count"), is(equalTo(41.0)));
+        assertThat(stdDevMap.get("sum"), is(equalTo(159.0)));
+        assertThat(stdDevMap.get("min"), is(equalTo(1.0)));
+        assertThat(stdDevMap.get("max"), is(equalTo(5.0)));
+        assertThat(stdDevMap.get("avg"), is(equalTo(3.8780487804878048)));
+        assertThat(stdDevMap.get("sum_of_squares"), is(equalTo(711.0)));
+        assertThat(stdDevMap.get("variance"), is(equalTo(2.3022010707911953)));
+        assertThat(stdDevMap.get("variance_population"), is(equalTo(2.3022010707911953)));
+        assertThat(stdDevMap.get("variance_sampling"), is(equalTo(2.3597560975609753)));
+        assertThat(stdDevMap.get("std_deviation"), is(equalTo(1.5173005868288574)));
+        assertThat(stdDevMap.get("std_deviation_population"), is(equalTo(1.5173005868288574)));
+        assertThat(stdDevMap.get("std_deviation_sampling"), is(equalTo(1.5361497640402693)));
+        assertThat(stdDevMap.get("std_upper"), is(equalTo(6.91264995414552)));
+        assertThat(stdDevMap.get("std_lower"), is(equalTo(0.84344760683009)));
+        assertThat(stdDevMap.get("std_upper_population"), is(equalTo(6.91264995414552)));
+        assertThat(stdDevMap.get("std_lower_population"), is(equalTo(0.84344760683009)));
+        assertThat(stdDevMap.get("std_upper_sampling"), is(equalTo(6.950348308568343)));
+        assertThat(stdDevMap.get("std_lower_sampling"), is(equalTo(0.8057492524072662)));
+    }
+
     public void testPivotWithBoxplot() throws Exception {
         String transformId = "boxplot_transform";
         String transformIndex = "boxplot_pivot_reviews";

x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/TransformAggregations.java

@@ -60,7 +60,6 @@ public final class TransformAggregations {
         "date_histogram",
         "date_range",
         "diversified_sampler",
-        "extended_stats", // https://github.com/elastic/elasticsearch/issues/51925
         "filters",
         "geo_distance",
         "geohash_grid",
@@ -120,7 +119,8 @@ enum AggregationType {
         MISSING("missing", LONG),
         TOP_METRICS("top_metrics", SOURCE),
         STATS("stats", DOUBLE),
-        BOXPLOT("boxplot", DOUBLE);
+        BOXPLOT("boxplot", DOUBLE),
+        EXTENDED_STATS("extended_stats", DOUBLE);
 
         private final String aggregationType;
         private final String targetMapping;

x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/TransformAggregationsTests.java

@@ -18,6 +18,7 @@
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.ExtendedStatsAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.MinAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.PercentilesAggregationBuilder;
@@ -31,7 +32,9 @@
 import java.util.Map;
 import java.util.stream.Collectors;
 
+import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.is;
 
 public class TransformAggregationsTests extends ESTestCase {
@@ -137,6 +140,9 @@ public void testResolveTargetMapping() {
         assertEquals("double", TransformAggregations.resolveTargetMapping("stats", null));
         assertEquals("double", TransformAggregations.resolveTargetMapping("stats", "int"));
 
+        // extended stats
+        assertEquals("double", TransformAggregations.resolveTargetMapping("extended_stats", "double"));
+
         // boxplot
         assertEquals("double", TransformAggregations.resolveTargetMapping("boxplot", "double"));
 
@@ -220,6 +226,37 @@ public void testGetAggregationOutputTypesStats() {
         assertEquals("stats", outputTypes.get("stats.sum"));
     }
 
+    public void testGetAggregationOutputTypesExtendedStats() {
+        var extendedStatsAggregationBuilder = new ExtendedStatsAggregationBuilder("extended_stats");
+
+        var inputAndOutputTypes = TransformAggregations.getAggregationInputAndOutputTypes(extendedStatsAggregationBuilder);
+        var outputTypes = inputAndOutputTypes.v2();
+        assertEquals(18, outputTypes.size());
+        assertThat(
+            outputTypes,
+            allOf(
+                hasEntry("extended_stats.std_upper_population", "extended_stats"),
+                hasEntry("extended_stats.variance", "extended_stats"),
+                hasEntry("extended_stats.avg", "extended_stats"),
+                hasEntry("extended_stats.min", "extended_stats"),
+                hasEntry("extended_stats.sum_of_squares", "extended_stats"),
+                hasEntry("extended_stats.sum", "extended_stats"),
+                hasEntry("extended_stats.variance_sampling", "extended_stats"),
+                hasEntry("extended_stats.std_lower_population", "extended_stats"),
+                hasEntry("extended_stats.std_deviation", "extended_stats"),
+                hasEntry("extended_stats.std_upper_sampling", "extended_stats"),
+                hasEntry("extended_stats.std_deviation_population", "extended_stats"),
+                hasEntry("extended_stats.max", "extended_stats"),
+                hasEntry("extended_stats.std_lower_sampling", "extended_stats"),
+                hasEntry("extended_stats.std_deviation_sampling", "extended_stats"),
+                hasEntry("extended_stats.std_upper", "extended_stats"),
+                hasEntry("extended_stats.count", "extended_stats"),
+                hasEntry("extended_stats.variance_population", "extended_stats"),
+                hasEntry("extended_stats.std_lower", "extended_stats")
+            )
+        );
+    }
+
     public void testGetAggregationOutputTypesRange() {
         {
             AggregationBuilder rangeAggregationBuilder = new RangeAggregationBuilder("range_agg_name").addUnboundedTo(100)