[OPIK-622] Remove deprecated dataset item fields

apps/opik-backend/src/main/java/com/comet/opik/api/DatasetItem.java

@@ -25,12 +25,6 @@
 public record DatasetItem(
         @JsonView( {
                 DatasetItem.View.Public.class, DatasetItem.View.Write.class}) UUID id,
-        @JsonView({DatasetItem.View.Public.class,
-                DatasetItem.View.Write.class}) @Schema(deprecated = true, description = "to be deprecated soon, please use data field") JsonNode input,
-        @JsonView({DatasetItem.View.Public.class,
-                DatasetItem.View.Write.class}) @Schema(deprecated = true, description = "to be deprecated soon, please use data field") JsonNode expectedOutput,
-        @JsonView({DatasetItem.View.Public.class,
-                DatasetItem.View.Write.class}) @Schema(deprecated = true, description = "to be deprecated soon, please use data field") JsonNode metadata,
         @JsonView({DatasetItem.View.Public.class, DatasetItem.View.Write.class}) UUID traceId,
         @JsonView({DatasetItem.View.Public.class, DatasetItem.View.Write.class}) UUID spanId,
         @JsonView({DatasetItem.View.Public.class, DatasetItem.View.Write.class}) @NotNull DatasetItemSource source,

apps/opik-backend/src/main/java/com/comet/opik/api/validate/DatasetItemInputValidator.java

@@ -9,12 +9,12 @@ public class DatasetItemInputValidator implements ConstraintValidator<DatasetIte
 
     @Override
     public boolean isValid(DatasetItem datasetItem, ConstraintValidatorContext context) {
-        boolean result = datasetItem.input() != null || MapUtils.isNotEmpty(datasetItem.data());
+        boolean result = MapUtils.isNotEmpty(datasetItem.data());
 
         if (!result) {
             context.disableDefaultConstraintViolation();
-            context.buildConstraintViolationWithTemplate("must provide either input or data field")
-                    .addPropertyNode("input")
+            context.buildConstraintViolationWithTemplate("must provide data field")
+                    .addPropertyNode("data")
                     .addConstraintViolation();
         }
 

apps/opik-backend/src/main/java/com/comet/opik/domain/DatasetItemDAO.java

@@ -74,10 +74,7 @@ INSERT INTO dataset_items (
                     source,
                     trace_id,
                     span_id,
-                    input,
                     data,
-                    expected_output,
-                    metadata,
                     created_at,
                     workspace_id,
                     created_by,
@@ -91,10 +88,7 @@ INSERT INTO dataset_items (
                              :source<item.index>,
                              :traceId<item.index>,
                              :spanId<item.index>,
-                             :input<item.index>,
                              :data<item.index>,
-                             :expectedOutput<item.index>,
-                             :metadata<item.index>,
                              now64(9),
                              :workspace_id,
                              :createdBy<item.index>,
@@ -144,10 +138,7 @@ INSERT INTO dataset_items (
                 SELECT
                     id,
                     dataset_id,
-                    input,
                     <if(truncate)> mapApply((k, v) -> (k, replaceRegexpAll(v, '<truncate>', '"[image]"')), data) as data <else> data <endif>,
-                    expected_output,
-                    metadata,
                     trace_id,
                     span_id,
                     source,
@@ -410,10 +401,7 @@ AND id IN (
             SELECT
                 di.id AS id,
                 di.dataset_id AS dataset_id,
-                di.input AS input,
                 <if(truncate)> mapApply((k, v) -> (k, replaceRegexpAll(v, '<truncate>', '"[image]"')), di.data) as data <else> di.data <endif>,
-                di.expected_output AS expected_output,
-                di.metadata AS metadata,
                 di.trace_id AS trace_id,
                 di.span_id AS span_id,
                 di.source AS source,
@@ -462,10 +450,7 @@ AND id IN (SELECT trace_id FROM experiment_items_final)
             GROUP BY
                 di.id,
                 di.dataset_id,
-                di.input,
                 di.data,
-                di.expected_output,
-                di.metadata,
                 di.trace_id,
                 di.span_id,
                 di.source,
@@ -571,28 +556,12 @@ private Mono<Long> mapAndInsert(
             for (DatasetItem item : items) {
                 Map<String, JsonNode> data = new HashMap<>(Optional.ofNullable(item.data()).orElse(Map.of()));
 
-                if (!data.containsKey("input") && item.input() != null) {
-                    data.put("input", item.input());
-                }
-
-                if (!data.containsKey("expected_output") && item.expectedOutput() != null) {
-                    data.put("expected_output", item.expectedOutput());
-                }
-
-                if (!data.containsKey("metadata") && item.metadata() != null) {
-                    data.put("metadata", item.metadata());
-                }
-
                 statement.bind("id" + i, item.id());
                 statement.bind("datasetId" + i, datasetId);
                 statement.bind("source" + i, item.source().getValue());
                 statement.bind("traceId" + i, DatasetItemResultMapper.getOrDefault(item.traceId()));
                 statement.bind("spanId" + i, DatasetItemResultMapper.getOrDefault(item.spanId()));
-                statement.bind("input" + i, DatasetItemResultMapper.getOrDefault(item.input()));
                 statement.bind("data" + i, DatasetItemResultMapper.getOrDefault(data));
-                statement.bind("expectedOutput" + i,
-                        DatasetItemResultMapper.getOrDefault(item.expectedOutput()));
-                statement.bind("metadata" + i, DatasetItemResultMapper.getOrDefault(item.metadata()));
                 statement.bind("createdBy" + i, userName);
                 statement.bind("lastUpdatedBy" + i, userName);
                 i++;

apps/opik-backend/src/main/java/com/comet/opik/domain/DatasetItemResultMapper.java

@@ -105,28 +105,9 @@ static Publisher<DatasetItem> mapItem(Result results) {
 
             Map<String, JsonNode> data = getData(row);
 
-            JsonNode input = getJsonNode(row, data, "input");
-            JsonNode expectedOutput = getJsonNode(row, data, "expected_output");
-            JsonNode metadata = getJsonNode(row, data, "metadata");
-
-            if (!data.containsKey("input")) {
-                data.put("input", input);
-            }
-
-            if (!data.containsKey("expected_output")) {
-                data.put("expected_output", expectedOutput);
-            }
-
-            if (!data.containsKey("metadata")) {
-                data.put("metadata", metadata);
-            }
-
             return DatasetItem.builder()
                     .id(row.get("id", UUID.class))
-                    .input(input)
                     .data(data)
-                    .expectedOutput(expectedOutput)
-                    .metadata(metadata)
                     .source(DatasetItemSource.fromString(row.get("source", String.class)))
                     .traceId(Optional.ofNullable(row.get("trace_id", String.class))
                             .filter(s -> !s.isBlank())

apps/opik-backend/src/test/java/com/comet/opik/api/resources/v1/priv/DatasetsResourceTest.java

@@ -99,7 +99,6 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Comparator;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -2770,18 +2769,16 @@ public Stream<Arguments> invalidDatasetItems() {
             return Stream.of(
                     arguments(factory.manufacturePojo(DatasetItemBatch.class).toBuilder()
                             .items(List.of(factory.manufacturePojo(DatasetItem.class).toBuilder()
-                                    .input(null)
                                     .data(null)
                                     .build()))
                             .build(),
-                            "items[0].input must provide either input or data field"),
+                            "items[0].data must provide data field"),
                     arguments(factory.manufacturePojo(DatasetItemBatch.class).toBuilder()
                             .items(List.of(factory.manufacturePojo(DatasetItem.class).toBuilder()
-                                    .input(null)
                                     .data(Map.of())
                                     .build()))
                             .build(),
-                            "items[0].input must provide either input or data field"),
+                            "items[0].data must provide data field"),
                     arguments(factory.manufacturePojo(DatasetItemBatch.class).toBuilder()
                             .items(List.of(factory.manufacturePojo(DatasetItem.class).toBuilder()
                                     .source(null)
@@ -2973,28 +2970,10 @@ void createDatasetItem__whenDatasetItemWorkspaceAndSpanWorkspaceDoesNotMatch__th
             }
         }
 
-        @Test
-        @DisplayName("when data is null, the accept the request")
-        void create__whenDataIsNull__thenAcceptTheRequest() {
-            var item = factory.manufacturePojo(DatasetItem.class).toBuilder()
-                    .data(null)
-                    .build();
-
-            var batch = factory.manufacturePojo(DatasetItemBatch.class).toBuilder()
-                    .items(List.of(item))
-                    .datasetId(null)
-                    .build();
-
-            putAndAssert(batch, TEST_WORKSPACE, API_KEY);
-
-            getItemAndAssert(item, TEST_WORKSPACE, API_KEY);
-        }
-
         @Test
         @DisplayName("when input is null but data is present, the accept the request")
         void create__whenInputIsNullButDataIsPresent__thenAcceptTheRequest() {
             var item = factory.manufacturePojo(DatasetItem.class).toBuilder()
-                    .input(null)
                     .build();
 
             var batch = factory.manufacturePojo(DatasetItemBatch.class).toBuilder()
@@ -3160,7 +3139,6 @@ void streamDataItems__whenStreamingHasMaxSize__thenReturnItemsSortedByCreatedDat
             var items = IntStream.range(0, 1000)
                     .mapToObj(i -> factory.manufacturePojo(DatasetItem.class).toBuilder()
                             .experimentItems(null)
-                            .metadata(null)
                             .createdAt(null)
                             .lastUpdatedAt(null)
                             .build())
@@ -3232,8 +3210,6 @@ private void getItemAndAssert(DatasetItem expectedDatasetItem, String workspaceN
         Map<String, JsonNode> data = Optional.ofNullable(expectedDatasetItem.data())
                 .orElse(Map.of());
 
-        expectedDatasetItem = mergeInputMap(expectedDatasetItem, data);
-
         assertThat(actualEntity.id()).isEqualTo(expectedDatasetItem.id());
         assertThat(actualEntity).usingRecursiveComparison()
                 .ignoringFields(IGNORED_FIELDS_DATA_ITEM)
@@ -3243,37 +3219,6 @@ private void getItemAndAssert(DatasetItem expectedDatasetItem, String workspaceN
         assertThat(actualEntity.lastUpdatedAt()).isInThePast();
     }
 
-    private DatasetItem mergeInputMap(DatasetItem expectedDatasetItem, Map<String, JsonNode> data) {
-
-        Map<String, JsonNode> newMap = new HashMap<>();
-
-        if (expectedDatasetItem.expectedOutput() != null) {
-            newMap.put("expected_output", expectedDatasetItem.expectedOutput());
-        }
-
-        if (expectedDatasetItem.input() != null) {
-            newMap.put("input", expectedDatasetItem.input());
-        }
-
-        if (expectedDatasetItem.metadata() != null) {
-            newMap.put("metadata", expectedDatasetItem.metadata());
-        }
-
-        Map<String, JsonNode> mergedMap = Stream
-                .concat(data.entrySet().stream(), newMap.entrySet().stream())
-                .collect(toMap(
-                        Map.Entry::getKey,
-                        Map.Entry::getValue,
-                        (v1, v2) -> v2 // In case of conflict, use the value from map2
-                ));
-
-        expectedDatasetItem = expectedDatasetItem.toBuilder()
-                .data(new HashMap<>(mergedMap))
-                .build();
-
-        return expectedDatasetItem;
-    }
-
     @Nested
     @DisplayName("Delete items:")
     @TestInstance(TestInstance.Lifecycle.PER_CLASS)
@@ -3474,7 +3419,10 @@ void getDatasetItemsByDatasetId__whenItemsWereUpdated__thenReturnCorrectItemsCou
 
             var updatedItems = items
                     .stream()
-                    .map(item -> item.toBuilder().input(factory.manufacturePojo(JsonNode.class)).build())
+                    .map(item -> item.toBuilder()
+                            .data(Map.of(factory.manufacturePojo(String.class),
+                                    factory.manufacturePojo(JsonNode.class)))
+                            .build())
                     .toList();
 
             var updatedBatch = batch.toBuilder()
@@ -3646,8 +3594,6 @@ private void assertPage(List<DatasetItem> expectedItems, List<DatasetItem> actua
             Map<String, JsonNode> data = Optional.ofNullable(expectedDatasetItem.data())
                     .orElse(Map.of());
 
-            expectedDatasetItem = mergeInputMap(expectedDatasetItem, data);
-
             assertThat(actualDatasetItem.data()).isEqualTo(expectedDatasetItem.data());
         }
     }
@@ -4137,12 +4083,6 @@ Stream<Arguments> find__whenFilteringBySupportedFields__thenReturnMatchingRows()
                             FieldType.FEEDBACK_SCORES_NUMBER, Operator.EQUAL, "sql_cost", "10")),
                     arguments(new ExperimentsComparisonFilter("feedback_scores",
                             FieldType.FEEDBACK_SCORES_NUMBER, Operator.NOT_EQUAL, "sql_cost", "10")),
-                    arguments(new ExperimentsComparisonFilter("output",
-                            FieldType.STRING, Operator.CONTAINS, null, "sql_cost")),
-                    arguments(new ExperimentsComparisonFilter("expected_output",
-                            FieldType.DICTIONARY, Operator.CONTAINS, "output", "sql_cost")),
-                    arguments(new ExperimentsComparisonFilter("metadata",
-                            FieldType.DICTIONARY, Operator.EQUAL, "sql_cost", "10")),
                     arguments(new ExperimentsComparisonFilter("meta_field",
                             FieldType.DICTIONARY, Operator.CONTAINS, "version[*]", "10")),
                     arguments(new ExperimentsComparisonFilter("releases",
@@ -4197,8 +4137,6 @@ private void createTraces(List<DatasetItem> items, String projectName, String wo
                 var item = items.get(i);
                 var trace = Trace.builder()
                         .id(GENERATOR.generate())
-                        .input(item.input())
-                        .output(item.expectedOutput())
                         .projectName(projectName)
                         .startTime(Instant.now())
                         .name("trace-" + i)
@@ -4214,12 +4152,6 @@ private void createDatasetItems(List<DatasetItem> items) {
                 if (i == 0) {
                     DatasetItem item = factory.manufacturePojo(DatasetItem.class)
                             .toBuilder()
-                            .input(JsonUtils
-                                    .getJsonNodeFromString(JsonUtils.writeValueAsString(Map.of("input", "sql_cost"))))
-                            .expectedOutput(JsonUtils
-                                    .getJsonNodeFromString(JsonUtils.writeValueAsString(Map.of("output", "sql_cost"))))
-                            .metadata(JsonUtils
-                                    .getJsonNodeFromString(JsonUtils.writeValueAsString(Map.of("sql_cost", 10))))
                             .source(DatasetItemSource.SDK)
                             .data(Map.of(
                                     "sql_tag", JsonUtils.readTree("sql_test"),
@@ -4748,18 +4680,12 @@ void setUp() {
                     .flatMap(connection -> Mono.from(connection.createStatement("""
                             INSERT INTO %s.%s (
                             id,
-                            input,
-                            expected_output,
-                            metadata,
                             source,
                             dataset_id,
                             workspace_id
-                            ) VALUES (:id, :input, :expected_output, :metadata, :source, :dataset_id, :workspace_id)
+                            ) VALUES (:id, :source, :dataset_id, :workspace_id)
                             """.formatted(DATABASE_NAME, "dataset_items"))
                             .bind("id", datasetItem.id())
-                            .bind("input", datasetItem.input().toString())
-                            .bind("expected_output", datasetItem.expectedOutput().toString())
-                            .bind("metadata", datasetItem.metadata().toString())
                             .bind("source", DatasetItemSource.SDK.getValue())
                             .bind("dataset_id", datasetId)
                             .bind("workspace_id", WORKSPACE_ID)
@@ -4768,22 +4694,6 @@ void setUp() {
 
             this.datasetItem = datasetItem;
         }
-
-        @Test
-        void findById__whenDatasetItemNotMigrated__thenReturnDatasetItemWithData() {
-
-            var item = datasetItem.toBuilder()
-                    .spanId(null)
-                    .traceId(null)
-                    .experimentItems(null)
-                    .source(DatasetItemSource.SDK)
-                    .data(Map.of("input", datasetItem.input(),
-                            "expected_output", datasetItem.expectedOutput(),
-                            "metadata", datasetItem.metadata()))
-                    .build();
-
-            getItemAndAssert(item, TEST_WORKSPACE, API_KEY);
-        }
     }
 
     private Set<Column> addDeprecatedFields(List<Map<String, JsonNode>> data) {
@@ -4799,12 +4709,6 @@ private Set<Column> addDeprecatedFields(List<Map<String, JsonNode>> data) {
                         .build())
                 .collect(Collectors.toCollection(HashSet::new));
 
-        columns.add(Column.builder().name("input").types(Set.of(ColumnType.OBJECT)).filterFieldPrefix("data").build());
-        columns.add(Column.builder().name("expected_output").types(Set.of(ColumnType.OBJECT)).filterFieldPrefix("data")
-                .build());
-        columns.add(
-                Column.builder().name("metadata").types(Set.of(ColumnType.OBJECT)).filterFieldPrefix("data").build());
-
         Map<String, Set<ColumnType>> results = columns.stream()
                 .collect(groupingBy(Column::name, mapping(Column::types, flatMapping(Set::stream, toSet()))));
 

apps/opik-backend/src/test/java/com/comet/opik/podam/manufacturer/DatasetItemTypeManufacturer.java

@@ -55,9 +55,6 @@ public DatasetItem getType(DataProviderStrategy strategy, AttributeMetadata meta
                 .traceId(traceId)
                 .spanId(spanId)
                 .id(strategy.getTypeValue(metadata, context, UUID.class))
-                .input(strategy.getTypeValue(metadata, context, JsonNode.class))
-                .expectedOutput(strategy.getTypeValue(metadata, context, JsonNode.class))
-                .metadata(strategy.getTypeValue(metadata, context, JsonNode.class))
                 .data(data)
                 .createdAt(Instant.now())
                 .lastUpdatedAt(Instant.now())