diff --git a/docs/changelog/118804.yaml b/docs/changelog/118804.yaml new file mode 100644 index 0000000000000..1548367a5485f --- /dev/null +++ b/docs/changelog/118804.yaml @@ -0,0 +1,15 @@ +pr: 118804 +summary: Add new experimental `rank_vectors` mapping for late-interaction second order + ranking +area: Vector Search +type: feature +issues: [] +highlight: + title: Add new experimental `rank_vectors` mapping for late-interaction second order + ranking + body: + Late-interaction models are powerful rerankers. While their size and overall + cost doesn't lend itself for HNSW indexing, utilizing them as second order reranking + can provide excellent boosts in relevance. The new `rank_vectors` mapping allows for rescoring + over new and novel multi-vector late-interaction models like ColBERT or ColPali. + notable: true diff --git a/docs/changelog/119601.yaml b/docs/changelog/119601.yaml new file mode 100644 index 0000000000000..3570588a5c690 --- /dev/null +++ b/docs/changelog/119601.yaml @@ -0,0 +1,13 @@ +pr: 119601 +summary: "[8.x] Add new experimental `rank_vectors` mapping for late-interaction second\ + \ order ranking" +area: Vector Search +type: feature +issues: [] +highlight: + title: "[8.x] Add new experimental `rank_vectors` mapping for late-interaction second\ + \ order ranking" + body: |- + Backports the following commits to 8.x: - Add new experimental + rank_vectors mapping for late-interaction second order ranking (#118804) + notable: true diff --git a/docs/reference/mapping/types.asciidoc b/docs/reference/mapping/types.asciidoc index babe4f508b5f0..e5155b7d4ce5b 100644 --- a/docs/reference/mapping/types.asciidoc +++ b/docs/reference/mapping/types.asciidoc @@ -180,6 +180,8 @@ include::types/rank-feature.asciidoc[] include::types/rank-features.asciidoc[] +include::types/rank-vectors.asciidoc[] + include::types/search-as-you-type.asciidoc[] include::types/semantic-text.asciidoc[] diff --git a/docs/reference/mapping/types/dense-vector.asciidoc b/docs/reference/mapping/types/dense-vector.asciidoc index e6e11d6dd539f..b56704b896706 100644 --- a/docs/reference/mapping/types/dense-vector.asciidoc +++ b/docs/reference/mapping/types/dense-vector.asciidoc @@ -1,4 +1,3 @@ -[role="xpack"] [[dense-vector]] === Dense vector field type ++++ diff --git a/docs/reference/mapping/types/rank-vectors.asciidoc b/docs/reference/mapping/types/rank-vectors.asciidoc new file mode 100644 index 0000000000000..a718a5e47ec85 --- /dev/null +++ b/docs/reference/mapping/types/rank-vectors.asciidoc @@ -0,0 +1,201 @@ +[role="xpack"] +[[rank-vectors]] +=== Rank Vectors +++++ + Rank Vectors +++++ +experimental::[] + +The `rank_vectors` field type enables late-interaction dense vector scoring in Elasticsearch. The number of vectors +per field can vary, but they must all share the same number of dimensions and element type. + +The purpose of vectors stored in this field is second order ranking documents with max-sim similarity. + +Here is a simple example of using this field with `float` elements. + +[source,console] +-------------------------------------------------- +PUT my-rank-vectors-float +{ + "mappings": { + "properties": { + "my_vector": { + "type": "rank_vectors" + } + } + } +} + +PUT my-rank-vectors-float/_doc/1 +{ + "my_vector" : [[0.5, 10, 6], [-0.5, 10, 10]] +} + +-------------------------------------------------- +// TESTSETUP + +In addition to the `float` element type, `byte` and `bit` element types are also supported. + +Here is an example of using this field with `byte` elements. + +[source,console] +-------------------------------------------------- +PUT my-rank-vectors-byte +{ + "mappings": { + "properties": { + "my_vector": { + "type": "rank_vectors", + "element_type": "byte" + } + } + } +} + +PUT my-rank-vectors-byte/_doc/1 +{ + "my_vector" : [[1, 2, 3], [4, 5, 6]] +} +-------------------------------------------------- + +Here is an example of using this field with `bit` elements. + +[source,console] +-------------------------------------------------- +PUT my-rank-vectors-bit +{ + "mappings": { + "properties": { + "my_vector": { + "type": "rank_vectors", + "element_type": "bit" + } + } + } +} + +POST /my-rank-vectors-bit/_bulk?refresh +{"index": {"_id" : "1"}} +{"my_vector": [127, -127, 0, 1, 42]} +{"index": {"_id" : "2"}} +{"my_vector": "8100012a7f"} +-------------------------------------------------- + +[role="child_attributes"] +[[rank-vectors-params]] +==== Parameters for rank vectors fields + +The `rank_vectors` field type supports the following parameters: + +[[rank-vectors-element-type]] +`element_type`:: +(Optional, string) +The data type used to encode vectors. The supported data types are +`float` (default), `byte`, and bit. + +.Valid values for `element_type` +[%collapsible%open] +==== +`float`::: +indexes a 4-byte floating-point +value per dimension. This is the default value. + +`byte`::: +indexes a 1-byte integer value per dimension. + +`bit`::: +indexes a single bit per dimension. Useful for very high-dimensional vectors or models that specifically support bit vectors. +NOTE: when using `bit`, the number of dimensions must be a multiple of 8 and must represent the number of bits. + +==== + +`dims`:: +(Optional, integer) +Number of vector dimensions. Can't exceed `4096`. If `dims` is not specified, +it will be set to the length of the first vector added to the field. + +[[rank-vectors-synthetic-source]] +==== Synthetic `_source` + +IMPORTANT: Synthetic `_source` is Generally Available only for TSDB indices +(indices that have `index.mode` set to `time_series`). For other indices +synthetic `_source` is in technical preview. Features in technical preview may +be changed or removed in a future release. Elastic will work to fix +any issues, but features in technical preview are not subject to the support SLA +of official GA features. + +`rank_vectors` fields support <> . + +[[rank-vectors-scoring]] +==== Scoring with rank vectors + +Rank vectors can be accessed and used in <>. + +For example, the following query scores documents based on the maxSim similarity between the query vector and the vectors stored in the `my_vector` field: + +[source,console] +-------------------------------------------------- +GET my-rank-vectors-float/_search +{ + "query": { + "script_score": { + "query": { + "match_all": {} + }, + "script": { + "source": "maxSimDotProduct(params.query_vector, 'my_vector')", + "params": { + "query_vector": [[0.5, 10, 6], [-0.5, 10, 10]] + } + } + } + } +} +-------------------------------------------------- + +Additionally, asymmetric similarity functions can be used to score against `bit` vectors. For example, the following query scores documents based on the maxSimDotProduct similarity between a floating point query vector and bit vectors stored in the `my_vector` field: + +[source,console] +-------------------------------------------------- +PUT my-rank-vectors-bit +{ + "mappings": { + "properties": { + "my_vector": { + "type": "rank_vectors", + "element_type": "bit" + } + } + } +} + +POST /my-rank-vectors-bit/_bulk?refresh +{"index": {"_id" : "1"}} +{"my_vector": [127, -127, 0, 1, 42]} +{"index": {"_id" : "2"}} +{"my_vector": "8100012a7f"} + +GET my-rank-vectors-bit/_search +{ + "query": { + "script_score": { + "query": { + "match_all": {} + }, + "script": { + "source": "maxSimDotProduct(params.query_vector, 'my_vector')", + "params": { + "query_vector": [ + [0.35, 0.77, 0.95, 0.15, 0.11, 0.08, 0.58, 0.06, 0.44, 0.52, 0.21, + 0.62, 0.65, 0.16, 0.64, 0.39, 0.93, 0.06, 0.93, 0.31, 0.92, 0.0, + 0.66, 0.86, 0.92, 0.03, 0.81, 0.31, 0.2 , 0.92, 0.95, 0.64, 0.19, + 0.26, 0.77, 0.64, 0.78, 0.32, 0.97, 0.84] + ] <1> + } + } + } + } +} +-------------------------------------------------- +<1> Note that the query vector has 40 elements, matching the number of bits in the bit vectors. + diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.score.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.score.txt index c08300f4351d5..fd8c81464b4a8 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.score.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.score.txt @@ -50,7 +50,5 @@ static_import { double cosineSimilarity(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.script.VectorScoreScriptUtils$CosineSimilarity double dotProduct(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.script.VectorScoreScriptUtils$DotProduct double hamming(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.script.VectorScoreScriptUtils$Hamming - double maxSimDotProduct(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.script.RankVectorsScoreScriptUtils$MaxSimDotProduct - double maxSimInvHamming(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.script.RankVectorsScoreScriptUtils$MaxSimInvHamming } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index b206c503a2739..4a22a53a64b41 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -101,7 +101,7 @@ public class DenseVectorFieldMapper extends FieldMapper { public static final String COSINE_MAGNITUDE_FIELD_SUFFIX = "._magnitude"; private static final float EPS = 1e-3f; - static final int BBQ_MIN_DIMS = 64; + public static final int BBQ_MIN_DIMS = 64; public static boolean isNotUnitVector(float magnitude) { return Math.abs(magnitude - 1.0f) > EPS; @@ -485,8 +485,12 @@ private VectorData parseHexEncodedVector( } @Override - VectorData parseKnnVector(DocumentParserContext context, int dims, IntBooleanConsumer dimChecker, VectorSimilarity similarity) - throws IOException { + public VectorData parseKnnVector( + DocumentParserContext context, + int dims, + IntBooleanConsumer dimChecker, + VectorSimilarity similarity + ) throws IOException { XContentParser.Token token = context.parser().currentToken(); return switch (token) { case START_ARRAY -> parseVectorArray(context, dims, dimChecker, similarity); @@ -516,17 +520,17 @@ public void parseKnnVectorAndIndex(DocumentParserContext context, DenseVectorFie } @Override - int getNumBytes(int dimensions) { + public int getNumBytes(int dimensions) { return dimensions; } @Override - ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes) { + public ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes) { return ByteBuffer.wrap(new byte[numBytes]); } @Override - int parseDimensionCount(DocumentParserContext context) throws IOException { + public int parseDimensionCount(DocumentParserContext context) throws IOException { XContentParser.Token currentToken = context.parser().currentToken(); return switch (currentToken) { case START_ARRAY -> { @@ -690,8 +694,12 @@ && isNotUnitVector(squaredMagnitude)) { } @Override - VectorData parseKnnVector(DocumentParserContext context, int dims, IntBooleanConsumer dimChecker, VectorSimilarity similarity) - throws IOException { + public VectorData parseKnnVector( + DocumentParserContext context, + int dims, + IntBooleanConsumer dimChecker, + VectorSimilarity similarity + ) throws IOException { int index = 0; float squaredMagnitude = 0; float[] vector = new float[dims]; @@ -710,12 +718,12 @@ VectorData parseKnnVector(DocumentParserContext context, int dims, IntBooleanCon } @Override - int getNumBytes(int dimensions) { + public int getNumBytes(int dimensions) { return dimensions * Float.BYTES; } @Override - ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes) { + public ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes) { return indexVersion.onOrAfter(LITTLE_ENDIAN_FLOAT_STORED_INDEX_VERSION) ? ByteBuffer.wrap(new byte[numBytes]).order(ByteOrder.LITTLE_ENDIAN) : ByteBuffer.wrap(new byte[numBytes]); @@ -888,8 +896,12 @@ private VectorData parseHexEncodedVector(DocumentParserContext context, IntBoole } @Override - VectorData parseKnnVector(DocumentParserContext context, int dims, IntBooleanConsumer dimChecker, VectorSimilarity similarity) - throws IOException { + public VectorData parseKnnVector( + DocumentParserContext context, + int dims, + IntBooleanConsumer dimChecker, + VectorSimilarity similarity + ) throws IOException { XContentParser.Token token = context.parser().currentToken(); return switch (token) { case START_ARRAY -> parseVectorArray(context, dims, dimChecker, similarity); @@ -919,18 +931,18 @@ public void parseKnnVectorAndIndex(DocumentParserContext context, DenseVectorFie } @Override - int getNumBytes(int dimensions) { + public int getNumBytes(int dimensions) { assert dimensions % Byte.SIZE == 0; return dimensions / Byte.SIZE; } @Override - ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes) { + public ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes) { return ByteBuffer.wrap(new byte[numBytes]); } @Override - int parseDimensionCount(DocumentParserContext context) throws IOException { + public int parseDimensionCount(DocumentParserContext context) throws IOException { XContentParser.Token currentToken = context.parser().currentToken(); return switch (currentToken) { case START_ARRAY -> { @@ -973,16 +985,16 @@ public void checkDimensions(Integer dvDims, int qvDims) { abstract void parseKnnVectorAndIndex(DocumentParserContext context, DenseVectorFieldMapper fieldMapper) throws IOException; - abstract VectorData parseKnnVector( + public abstract VectorData parseKnnVector( DocumentParserContext context, int dims, IntBooleanConsumer dimChecker, VectorSimilarity similarity ) throws IOException; - abstract int getNumBytes(int dimensions); + public abstract int getNumBytes(int dimensions); - abstract ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes); + public abstract ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes); public abstract void checkVectorBounds(float[] vector); @@ -1000,7 +1012,7 @@ public void checkDimensions(Integer dvDims, int qvDims) { } } - int parseDimensionCount(DocumentParserContext context) throws IOException { + public int parseDimensionCount(DocumentParserContext context) throws IOException { int index = 0; for (Token token = context.parser().nextToken(); token != Token.END_ARRAY; token = context.parser().nextToken()) { index++; @@ -1087,7 +1099,7 @@ public static ElementType fromString(String name) { } } - static final Map namesToElementType = Map.of( + public static final Map namesToElementType = Map.of( ElementType.BYTE.toString(), ElementType.BYTE, ElementType.FLOAT.toString(), @@ -2494,9 +2506,10 @@ public String fieldName() { } /** - * @FunctionalInterface for a function that takes a int and boolean + * Interface for a function that takes a int and boolean */ - interface IntBooleanConsumer { + @FunctionalInterface + public interface IntBooleanConsumer { void accept(int value, boolean isComplete); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsDVLeafFieldData.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsDVLeafFieldData.java deleted file mode 100644 index 0125d0249ec2b..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsDVLeafFieldData.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.index.mapper.vectors; - -import org.apache.lucene.index.BinaryDocValues; -import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.LeafReader; -import org.elasticsearch.index.fielddata.LeafFieldData; -import org.elasticsearch.index.fielddata.SortedBinaryDocValues; -import org.elasticsearch.script.field.DocValuesScriptFieldFactory; -import org.elasticsearch.script.field.vectors.BitRankVectorsDocValuesField; -import org.elasticsearch.script.field.vectors.ByteRankVectorsDocValuesField; -import org.elasticsearch.script.field.vectors.FloatRankVectorsDocValuesField; - -import java.io.IOException; - -final class RankVectorsDVLeafFieldData implements LeafFieldData { - private final LeafReader reader; - private final String field; - private final DenseVectorFieldMapper.ElementType elementType; - private final int dims; - - RankVectorsDVLeafFieldData(LeafReader reader, String field, DenseVectorFieldMapper.ElementType elementType, int dims) { - this.reader = reader; - this.field = field; - this.elementType = elementType; - this.dims = dims; - } - - @Override - public DocValuesScriptFieldFactory getScriptFieldFactory(String name) { - try { - BinaryDocValues values = DocValues.getBinary(reader, field); - BinaryDocValues magnitudeValues = DocValues.getBinary(reader, field + RankVectorsFieldMapper.VECTOR_MAGNITUDES_SUFFIX); - return switch (elementType) { - case BYTE -> new ByteRankVectorsDocValuesField(values, magnitudeValues, name, elementType, dims); - case FLOAT -> new FloatRankVectorsDocValuesField(values, magnitudeValues, name, elementType, dims); - case BIT -> new BitRankVectorsDocValuesField(values, magnitudeValues, name, elementType, dims); - }; - } catch (IOException e) { - throw new IllegalStateException("Cannot load doc values for multi-vector field!", e); - } - } - - @Override - public SortedBinaryDocValues getBytesValues() { - throw new UnsupportedOperationException("String representation of doc values for multi-vector fields is not supported"); - } - - @Override - public long ramBytesUsed() { - return 0; - } -} diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java index 3dc25b058b1d6..09be98630d5c4 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -67,7 +67,6 @@ import org.elasticsearch.index.mapper.VersionFieldMapper; import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; -import org.elasticsearch.index.mapper.vectors.RankVectorsFieldMapper; import org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapper; import org.elasticsearch.index.seqno.RetentionLeaseBackgroundSyncAction; import org.elasticsearch.index.seqno.RetentionLeaseSyncAction; @@ -211,9 +210,6 @@ public static Map getMappers(List mappe mappers.put(DenseVectorFieldMapper.CONTENT_TYPE, DenseVectorFieldMapper.PARSER); mappers.put(SparseVectorFieldMapper.CONTENT_TYPE, SparseVectorFieldMapper.PARSER); - if (RankVectorsFieldMapper.FEATURE_FLAG.isEnabled()) { - mappers.put(RankVectorsFieldMapper.CONTENT_TYPE, RankVectorsFieldMapper.PARSER); - } for (MapperPlugin mapperPlugin : mapperPlugins) { for (Map.Entry entry : mapperPlugin.getMappers().entrySet()) { diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java index 7bcdd523fd3d3..3d32c1ff6cf27 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java @@ -10,7 +10,6 @@ package org.elasticsearch.rest.action.search; import org.elasticsearch.Build; -import org.elasticsearch.index.mapper.vectors.RankVectorsFieldMapper; import java.util.HashSet; import java.util.Set; @@ -32,14 +31,8 @@ private SearchCapabilities() {} private static final String DENSE_VECTOR_DOCVALUE_FIELDS = "dense_vector_docvalue_fields"; /** Support kql query. */ private static final String KQL_QUERY_SUPPORTED = "kql_query"; - /** Support rank-vectors field mapper. */ - private static final String RANK_VECTORS_FIELD_MAPPER = "rank_vectors_field_mapper"; /** Support propagating nested retrievers' inner_hits to top-level compound retrievers . */ private static final String NESTED_RETRIEVER_INNER_HITS_SUPPORT = "nested_retriever_inner_hits_support"; - /** Support rank-vectors script field access. */ - private static final String RANK_VECTORS_SCRIPT_ACCESS = "rank_vectors_script_access"; - /** Initial support for rank-vectors maxSim functions access. */ - private static final String RANK_VECTORS_SCRIPT_MAX_SIM = "rank_vectors_script_max_sim_with_bugfix"; /** Fixed the math in {@code moving_fn}'s {@code linearWeightedAvg}. */ private static final String MOVING_FN_RIGHT_MATH = "moving_fn_right_math"; @@ -59,11 +52,6 @@ private SearchCapabilities() {} capabilities.add(OPTIMIZED_SCALAR_QUANTIZATION_BBQ); capabilities.add(KNN_QUANTIZED_VECTOR_RESCORE); capabilities.add(MOVING_FN_RIGHT_MATH); - if (RankVectorsFieldMapper.FEATURE_FLAG.isEnabled()) { - capabilities.add(RANK_VECTORS_FIELD_MAPPER); - capabilities.add(RANK_VECTORS_SCRIPT_ACCESS); - capabilities.add(RANK_VECTORS_SCRIPT_MAX_SIM); - } if (Build.current().isSnapshot()) { capabilities.add(KQL_QUERY_SUPPORTED); } diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/ByteRankVectorsDocValuesField.java b/server/src/main/java/org/elasticsearch/script/field/vectors/ByteRankVectorsDocValuesField.java index db81bb6ebe1cb..1bff1b50fb5ac 100644 --- a/server/src/main/java/org/elasticsearch/script/field/vectors/ByteRankVectorsDocValuesField.java +++ b/server/src/main/java/org/elasticsearch/script/field/vectors/ByteRankVectorsDocValuesField.java @@ -111,13 +111,13 @@ public boolean isEmpty() { return value == null; } - static class ByteVectorIterator implements VectorIterator { + public static class ByteVectorIterator implements VectorIterator { private final byte[] buffer; private final BytesRef vectorValues; private final int size; private int idx = 0; - ByteVectorIterator(BytesRef vectorValues, byte[] buffer, int size) { + public ByteVectorIterator(BytesRef vectorValues, byte[] buffer, int size) { assert vectorValues.length == (buffer.length * size); this.vectorValues = vectorValues; this.size = size; diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/FloatRankVectorsDocValuesField.java b/server/src/main/java/org/elasticsearch/script/field/vectors/FloatRankVectorsDocValuesField.java index 39bc1e621113b..d47795a3b2401 100644 --- a/server/src/main/java/org/elasticsearch/script/field/vectors/FloatRankVectorsDocValuesField.java +++ b/server/src/main/java/org/elasticsearch/script/field/vectors/FloatRankVectorsDocValuesField.java @@ -110,14 +110,14 @@ private void decodeVectorIfNecessary() { } } - static class FloatVectorIterator implements VectorIterator { + public static class FloatVectorIterator implements VectorIterator { private final float[] buffer; private final FloatBuffer vectorValues; private final BytesRef vectorValueBytesRef; private final int size; private int idx = 0; - FloatVectorIterator(BytesRef vectorValues, float[] buffer, int size) { + public FloatVectorIterator(BytesRef vectorValues, float[] buffer, int size) { assert vectorValues.length == (buffer.length * Float.BYTES * size); this.vectorValueBytesRef = vectorValues; this.vectorValues = ByteBuffer.wrap(vectorValues.bytes, vectorValues.offset, vectorValues.length) diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 4e7c98a8cac97..366dbe05cf7f1 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -110,7 +110,6 @@ import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; -import org.elasticsearch.index.mapper.vectors.RankVectorsFieldMapper; import org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapper; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.shard.IndexShard; @@ -203,7 +202,6 @@ public abstract class AggregatorTestCase extends ESTestCase { private static final List TYPE_TEST_BLACKLIST = List.of( ObjectMapper.CONTENT_TYPE, // Cannot aggregate objects DenseVectorFieldMapper.CONTENT_TYPE, // Cannot aggregate dense vectors - RankVectorsFieldMapper.CONTENT_TYPE, // Cannot aggregate dense vectors SparseVectorFieldMapper.CONTENT_TYPE, // Sparse vectors are no longer supported NestedObjectMapper.CONTENT_TYPE, // TODO support for nested diff --git a/x-pack/plugin/rank-vectors/build.gradle b/x-pack/plugin/rank-vectors/build.gradle new file mode 100644 index 0000000000000..53aabb8fdbf74 --- /dev/null +++ b/x-pack/plugin/rank-vectors/build.gradle @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +apply plugin: 'elasticsearch.internal-es-plugin' +apply plugin: 'elasticsearch.internal-cluster-test' + +esplugin { + name = 'rank-vectors' + description = 'Rank vectors in search.' + classname = 'org.elasticsearch.xpack.rank.vectors.RankVectorsPlugin' + extendedPlugins = ['x-pack-core', 'lang-painless'] +} + +dependencies { + compileOnly project(path: xpackModule('core')) + compileOnly(project(':modules:lang-painless:spi')) + + testImplementation(testArtifact(project(xpackModule('core')))) + testImplementation(testArtifact(project(':server'))) +} diff --git a/x-pack/plugin/rank-vectors/src/main/java/module-info.java b/x-pack/plugin/rank-vectors/src/main/java/module-info.java new file mode 100644 index 0000000000000..4af3c994edd38 --- /dev/null +++ b/x-pack/plugin/rank-vectors/src/main/java/module-info.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module org.elasticsearch.rank.vectors { + requires org.elasticsearch.xcore; + requires org.elasticsearch.painless.spi; + requires org.elasticsearch.server; + requires org.apache.lucene.core; + requires org.elasticsearch.xcontent; + + exports org.elasticsearch.xpack.rank.vectors; + exports org.elasticsearch.xpack.rank.vectors.mapper; + exports org.elasticsearch.xpack.rank.vectors.script; + + // whitelist resource access + opens org.elasticsearch.xpack.rank.vectors.script to org.elasticsearch.painless.spi; + + provides org.elasticsearch.painless.spi.PainlessExtension with org.elasticsearch.xpack.rank.vectors.script.RankVectorsPainlessExtension; + provides org.elasticsearch.features.FeatureSpecification with org.elasticsearch.xpack.rank.vectors.RankVectorsFeatures; + +} diff --git a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/RankVectorsFeatures.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/RankVectorsFeatures.java new file mode 100644 index 0000000000000..44b1b7a068860 --- /dev/null +++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/RankVectorsFeatures.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.rank.vectors; + +import org.elasticsearch.features.FeatureSpecification; +import org.elasticsearch.features.NodeFeature; + +import java.util.Set; + +public class RankVectorsFeatures implements FeatureSpecification { + public static final NodeFeature RANK_VECTORS_FEATURE = new NodeFeature("rank_vectors"); + + @Override + public Set getTestFeatures() { + return Set.of(RANK_VECTORS_FEATURE); + } + +} diff --git a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/RankVectorsPlugin.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/RankVectorsPlugin.java new file mode 100644 index 0000000000000..35c87f1fc1847 --- /dev/null +++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/RankVectorsPlugin.java @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.rank.vectors; + +import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.license.License; +import org.elasticsearch.license.LicenseUtils; +import org.elasticsearch.license.LicensedFeature; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.plugins.MapperPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.xpack.core.XPackPlugin; +import org.elasticsearch.xpack.rank.vectors.mapper.RankVectorsFieldMapper; + +import java.util.Map; + +import static org.elasticsearch.index.mapper.FieldMapper.notInMultiFields; +import static org.elasticsearch.xpack.rank.vectors.mapper.RankVectorsFieldMapper.CONTENT_TYPE; + +public class RankVectorsPlugin extends Plugin implements MapperPlugin { + public static final LicensedFeature.Momentary RANK_VECTORS_FEATURE = LicensedFeature.momentary( + null, + "rank-vectors", + License.OperationMode.ENTERPRISE + ); + + @Override + public Map getMappers() { + return Map.of(CONTENT_TYPE, new FieldMapper.TypeParser((n, c) -> { + if (RANK_VECTORS_FEATURE.check(getLicenseState()) == false) { + throw LicenseUtils.newComplianceException("Rank Vectors"); + } + return new RankVectorsFieldMapper.Builder(n, c.indexVersionCreated(), getLicenseState()); + }, notInMultiFields(CONTENT_TYPE))); + } + + protected XPackLicenseState getLicenseState() { + return XPackPlugin.getSharedLicenseState(); + } +} diff --git a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsDVLeafFieldData.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsDVLeafFieldData.java new file mode 100644 index 0000000000000..b858b935c1483 --- /dev/null +++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsDVLeafFieldData.java @@ -0,0 +1,158 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.rank.vectors.mapper; + +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.fielddata.FormattedDocValues; +import org.elasticsearch.index.fielddata.LeafFieldData; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; +import org.elasticsearch.script.field.DocValuesScriptFieldFactory; +import org.elasticsearch.script.field.vectors.BitRankVectorsDocValuesField; +import org.elasticsearch.script.field.vectors.ByteRankVectorsDocValuesField; +import org.elasticsearch.script.field.vectors.FloatRankVectorsDocValuesField; +import org.elasticsearch.script.field.vectors.VectorIterator; +import org.elasticsearch.search.DocValueFormat; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +final class RankVectorsDVLeafFieldData implements LeafFieldData { + private final LeafReader reader; + private final String field; + private final DenseVectorFieldMapper.ElementType elementType; + private final int dims; + + RankVectorsDVLeafFieldData(LeafReader reader, String field, DenseVectorFieldMapper.ElementType elementType, int dims) { + this.reader = reader; + this.field = field; + this.elementType = elementType; + this.dims = dims; + } + + @Override + public FormattedDocValues getFormattedValues(DocValueFormat format) { + int dims = elementType == DenseVectorFieldMapper.ElementType.BIT ? this.dims / Byte.SIZE : this.dims; + return switch (elementType) { + case BYTE, BIT -> new FormattedDocValues() { + private final byte[] vector = new byte[dims]; + private BytesRef ref = null; + private int numVecs = -1; + private final BinaryDocValues binary; + { + try { + binary = DocValues.getBinary(reader, field); + } catch (IOException e) { + throw new IllegalStateException("Cannot load doc values", e); + } + } + + @Override + public boolean advanceExact(int docId) throws IOException { + if (binary == null || binary.advanceExact(docId) == false) { + return false; + } + ref = binary.binaryValue(); + assert ref.length % dims == 0; + numVecs = ref.length / dims; + return true; + } + + @Override + public int docValueCount() { + return 1; + } + + public Object nextValue() { + // Boxed to keep from `byte[]` being transformed into a string + List vectors = new ArrayList<>(numVecs); + VectorIterator iterator = new ByteRankVectorsDocValuesField.ByteVectorIterator(ref, vector, numVecs); + while (iterator.hasNext()) { + byte[] v = iterator.next(); + Byte[] vec = new Byte[dims]; + for (int i = 0; i < dims; i++) { + vec[i] = v[i]; + } + vectors.add(vec); + } + return vectors; + } + }; + case FLOAT -> new FormattedDocValues() { + private final float[] vector = new float[dims]; + private BytesRef ref = null; + private int numVecs = -1; + private final BinaryDocValues binary; + { + try { + binary = DocValues.getBinary(reader, field); + } catch (IOException e) { + throw new IllegalStateException("Cannot load doc values", e); + } + } + + @Override + public boolean advanceExact(int docId) throws IOException { + if (binary == null || binary.advanceExact(docId) == false) { + return false; + } + ref = binary.binaryValue(); + assert ref.length % (Float.BYTES * dims) == 0; + numVecs = ref.length / (Float.BYTES * dims); + return true; + } + + @Override + public int docValueCount() { + return 1; + } + + @Override + public Object nextValue() { + List vectors = new ArrayList<>(numVecs); + VectorIterator iterator = new FloatRankVectorsDocValuesField.FloatVectorIterator(ref, vector, numVecs); + while (iterator.hasNext()) { + float[] v = iterator.next(); + vectors.add(Arrays.copyOf(v, v.length)); + } + return vectors; + } + }; + }; + } + + @Override + public DocValuesScriptFieldFactory getScriptFieldFactory(String name) { + try { + BinaryDocValues values = DocValues.getBinary(reader, field); + BinaryDocValues magnitudeValues = DocValues.getBinary(reader, field + RankVectorsFieldMapper.VECTOR_MAGNITUDES_SUFFIX); + return switch (elementType) { + case BYTE -> new ByteRankVectorsDocValuesField(values, magnitudeValues, name, elementType, dims); + case FLOAT -> new FloatRankVectorsDocValuesField(values, magnitudeValues, name, elementType, dims); + case BIT -> new BitRankVectorsDocValuesField(values, magnitudeValues, name, elementType, dims); + }; + } catch (IOException e) { + throw new IllegalStateException("Cannot load doc values for multi-vector field!", e); + } + } + + @Override + public SortedBinaryDocValues getBytesValues() { + throw new UnsupportedOperationException("String representation of doc values for multi-vector fields is not supported"); + } + + @Override + public long ramBytesUsed() { + return 0; + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldMapper.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java similarity index 88% rename from server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldMapper.java rename to x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java index d57dbf79b450c..873d67e76b04a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldMapper.java +++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.index.mapper.vectors; +package org.elasticsearch.xpack.rank.vectors.mapper; import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.index.BinaryDocValues; @@ -15,7 +13,6 @@ import org.apache.lucene.search.FieldExistsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.util.FeatureFlag; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.fielddata.FieldDataContext; @@ -31,7 +28,10 @@ import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.mapper.ValueFetcher; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.license.LicenseUtils; +import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.vectors.VectorData; @@ -50,11 +50,11 @@ import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT; import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT_BIT; import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.namesToElementType; +import static org.elasticsearch.xpack.rank.vectors.RankVectorsPlugin.RANK_VECTORS_FEATURE; public class RankVectorsFieldMapper extends FieldMapper { public static final String VECTOR_MAGNITUDES_SUFFIX = "._magnitude"; - public static final FeatureFlag FEATURE_FLAG = new FeatureFlag("rank_vectors"); public static final String CONTENT_TYPE = "rank_vectors"; private static RankVectorsFieldMapper toType(FieldMapper in) { @@ -111,10 +111,12 @@ public static class Builder extends FieldMapper.Builder { private final Parameter> meta = Parameter.metaParam(); private final IndexVersion indexCreatedVersion; + private final XPackLicenseState licenseState; - public Builder(String name, IndexVersion indexCreatedVersion) { + public Builder(String name, IndexVersion indexCreatedVersion, XPackLicenseState licenseState) { super(name); this.indexCreatedVersion = indexCreatedVersion; + this.licenseState = licenseState; } @Override @@ -122,18 +124,12 @@ protected Parameter[] getParameters() { return new Parameter[] { elementType, dims, meta }; } - public RankVectorsFieldMapper.Builder dimensions(int dimensions) { - this.dims.setValue(dimensions); - return this; - } - - public RankVectorsFieldMapper.Builder elementType(DenseVectorFieldMapper.ElementType elementType) { - this.elementType.setValue(elementType); - return this; - } - @Override public RankVectorsFieldMapper build(MapperBuilderContext context) { + // Validate on Mapping creation + if (RANK_VECTORS_FEATURE.check(licenseState) == false) { + throw LicenseUtils.newComplianceException("Rank Vectors"); + } // Validate again here because the dimensions or element type could have been set programmatically, // which affects index option validity validate(); @@ -143,36 +139,32 @@ public RankVectorsFieldMapper build(MapperBuilderContext context) { context.buildFullName(leafName()), elementType.getValue(), dims.getValue(), - indexCreatedVersion, + licenseState, meta.getValue() ), builderParams(this, context), - indexCreatedVersion + indexCreatedVersion, + licenseState ); } } - public static final TypeParser PARSER = new TypeParser( - (n, c) -> new RankVectorsFieldMapper.Builder(n, c.indexVersionCreated()), - notInMultiFields(CONTENT_TYPE) - ); - public static final class RankVectorsFieldType extends SimpleMappedFieldType { private final DenseVectorFieldMapper.ElementType elementType; private final Integer dims; - private final IndexVersion indexCreatedVersion; + private final XPackLicenseState licenseState; public RankVectorsFieldType( String name, DenseVectorFieldMapper.ElementType elementType, Integer dims, - IndexVersion indexCreatedVersion, + XPackLicenseState licenseState, Map meta ) { super(name, false, false, true, TextSearchInfo.NONE, meta); this.elementType = elementType; this.dims = dims; - this.indexCreatedVersion = indexCreatedVersion; + this.licenseState = licenseState; } @Override @@ -195,9 +187,7 @@ protected Object parseSourceValue(Object value) { @Override public DocValueFormat docValueFormat(String format, ZoneId timeZone) { - throw new IllegalArgumentException( - "Field [" + name() + "] of type [" + typeName() + "] doesn't support docvalue_fields or aggregations" - ); + return DocValueFormat.DENSE_VECTOR; } @Override @@ -207,7 +197,10 @@ public boolean isAggregatable() { @Override public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) { - return new RankVectorsIndexFieldData.Builder(name(), CoreValuesSourceType.KEYWORD, indexCreatedVersion, dims, elementType); + if (RANK_VECTORS_FEATURE.check(licenseState) == false) { + throw LicenseUtils.newComplianceException("Rank Vectors"); + } + return new RankVectorsIndexFieldData.Builder(name(), CoreValuesSourceType.KEYWORD, dims, elementType); } @Override @@ -230,10 +223,18 @@ DenseVectorFieldMapper.ElementType getElementType() { } private final IndexVersion indexCreatedVersion; - - private RankVectorsFieldMapper(String simpleName, MappedFieldType fieldType, BuilderParams params, IndexVersion indexCreatedVersion) { + private final XPackLicenseState licenseState; + + private RankVectorsFieldMapper( + String simpleName, + MappedFieldType fieldType, + BuilderParams params, + IndexVersion indexCreatedVersion, + XPackLicenseState licenseState + ) { super(simpleName, fieldType, params); this.indexCreatedVersion = indexCreatedVersion; + this.licenseState = licenseState; } @Override @@ -248,6 +249,9 @@ public boolean parsesArrayValue() { @Override public void parse(DocumentParserContext context) throws IOException { + if (RANK_VECTORS_FEATURE.check(licenseState) == false) { + throw LicenseUtils.newComplianceException("Rank Vectors"); + } if (context.doc().getByKey(fieldType().name()) != null) { throw new IllegalArgumentException( "Field [" @@ -281,10 +285,10 @@ public void parse(DocumentParserContext context) throws IOException { fieldType().name(), fieldType().elementType, currentDims, - indexCreatedVersion, + licenseState, fieldType().meta() ); - Mapper update = new RankVectorsFieldMapper(leafName(), updatedFieldType, builderParams, indexCreatedVersion); + Mapper update = new RankVectorsFieldMapper(leafName(), updatedFieldType, builderParams, indexCreatedVersion, licenseState); context.addDynamicMapper(update); return; } @@ -366,12 +370,12 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { - return new RankVectorsFieldMapper.Builder(leafName(), indexCreatedVersion).init(this); + return new Builder(leafName(), indexCreatedVersion, licenseState).init(this); } @Override protected SyntheticSourceSupport syntheticSourceSupport() { - return new SyntheticSourceSupport.Native(new RankVectorsFieldMapper.DocValuesSyntheticFieldLoader()); + return new SyntheticSourceSupport.Native(new DocValuesSyntheticFieldLoader()); } private class DocValuesSyntheticFieldLoader extends SourceLoader.DocValuesBasedSyntheticFieldLoader { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsIndexFieldData.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsIndexFieldData.java similarity index 76% rename from server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsIndexFieldData.java rename to x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsIndexFieldData.java index 7f54d2b9a8ad8..7ec426b904ce0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsIndexFieldData.java +++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsIndexFieldData.java @@ -1,20 +1,18 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.index.mapper.vectors; +package org.elasticsearch.xpack.rank.vectors.mapper; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.SortField; import org.elasticsearch.common.util.BigArrays; -import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.MultiValueMode; @@ -26,19 +24,16 @@ public class RankVectorsIndexFieldData implements IndexFieldData build(IndexFieldDataCache cache, CircuitBreakerService breakerService) { - return new RankVectorsIndexFieldData(name, dims, valuesSourceType, indexVersion, elementType); + return new RankVectorsIndexFieldData(name, dims, valuesSourceType, elementType); } } } diff --git a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsPainlessExtension.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsPainlessExtension.java new file mode 100644 index 0000000000000..5893452f5fdf8 --- /dev/null +++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsPainlessExtension.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.rank.vectors.script; + +import org.elasticsearch.painless.spi.PainlessExtension; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; +import org.elasticsearch.script.ScoreScript; +import org.elasticsearch.script.ScriptContext; + +import java.util.List; +import java.util.Map; + +public class RankVectorsPainlessExtension implements PainlessExtension { + private static final Whitelist WHITELIST = WhitelistLoader.loadFromResourceFiles( + RankVectorsPainlessExtension.class, + "rank_vector_whitelist.txt" + ); + + @Override + public Map, List> getContextWhitelists() { + return Map.of(ScoreScript.CONTEXT, List.of(WHITELIST)); + } +} diff --git a/server/src/main/java/org/elasticsearch/script/RankVectorsScoreScriptUtils.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsScoreScriptUtils.java similarity index 97% rename from server/src/main/java/org/elasticsearch/script/RankVectorsScoreScriptUtils.java rename to x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsScoreScriptUtils.java index 2d11641cb5aa7..3d692db08ff9e 100644 --- a/server/src/main/java/org/elasticsearch/script/RankVectorsScoreScriptUtils.java +++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsScoreScriptUtils.java @@ -1,16 +1,15 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.script; +package org.elasticsearch.xpack.rank.vectors.script; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; +import org.elasticsearch.script.ScoreScript; import org.elasticsearch.script.field.vectors.DenseVector; import org.elasticsearch.script.field.vectors.RankVectorsDocValuesField; diff --git a/x-pack/plugin/rank-vectors/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification b/x-pack/plugin/rank-vectors/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification new file mode 100644 index 0000000000000..388dbe7dead96 --- /dev/null +++ b/x-pack/plugin/rank-vectors/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification @@ -0,0 +1,8 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0; you may not use this file except in compliance with the Elastic License +# 2.0. +# + +org.elasticsearch.xpack.rank.vectors.RankVectorsFeatures diff --git a/x-pack/plugin/rank-vectors/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension b/x-pack/plugin/rank-vectors/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension new file mode 100644 index 0000000000000..06c77434c77a6 --- /dev/null +++ b/x-pack/plugin/rank-vectors/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension @@ -0,0 +1 @@ +org.elasticsearch.xpack.rank.vectors.script.RankVectorsPainlessExtension diff --git a/x-pack/plugin/rank-vectors/src/main/resources/org/elasticsearch/xpack/rank/vectors/script/rank_vector_whitelist.txt b/x-pack/plugin/rank-vectors/src/main/resources/org/elasticsearch/xpack/rank/vectors/script/rank_vector_whitelist.txt new file mode 100644 index 0000000000000..ce3d04f73dea8 --- /dev/null +++ b/x-pack/plugin/rank-vectors/src/main/resources/org/elasticsearch/xpack/rank/vectors/script/rank_vector_whitelist.txt @@ -0,0 +1,13 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0; you may not use this file except in compliance with the Elastic License +# 2.0. +# + +# This file contains a rank vectors whitelist for functions to be used in Score context + +static_import { + double maxSimDotProduct(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.xpack.rank.vectors.script.RankVectorsScoreScriptUtils$MaxSimDotProduct + double maxSimInvHamming(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.xpack.rank.vectors.script.RankVectorsScoreScriptUtils$MaxSimInvHamming +} diff --git a/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/LocalStateRankVectors.java b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/LocalStateRankVectors.java new file mode 100644 index 0000000000000..ba085b8f0d705 --- /dev/null +++ b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/LocalStateRankVectors.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.rank.vectors; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.license.License; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.license.internal.XPackLicenseStatus; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; + +import java.util.Map; + +public class LocalStateRankVectors extends LocalStateCompositeXPackPlugin { + + private final RankVectorsPlugin rankVectorsPlugin; + private final XPackLicenseState licenseState = new XPackLicenseState( + System::currentTimeMillis, + new XPackLicenseStatus(License.OperationMode.TRIAL, true, null) + ); + + public LocalStateRankVectors(Settings settings) { + super(settings, null); + LocalStateRankVectors thisVar = this; + rankVectorsPlugin = new RankVectorsPlugin() { + @Override + protected XPackLicenseState getLicenseState() { + return licenseState; + } + }; + } + + @Override + public Map getMappers() { + return rankVectorsPlugin.getMappers(); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldMapperTests.java b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java similarity index 97% rename from server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldMapperTests.java rename to x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java index e81c28cbcc444..25264976c58a5 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldMapperTests.java +++ b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.index.mapper.vectors; +package org.elasticsearch.xpack.rank.vectors.mapper; import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.index.IndexableField; @@ -29,18 +27,21 @@ import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType; import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.lookup.Source; import org.elasticsearch.search.lookup.SourceProvider; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.rank.vectors.LocalStateRankVectors; import org.junit.AssumptionViolatedException; -import org.junit.BeforeClass; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.FloatBuffer; import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Set; import java.util.stream.Stream; @@ -53,11 +54,6 @@ public class RankVectorsFieldMapperTests extends MapperTestCase { - @BeforeClass - public static void setup() { - assumeTrue("Requires rank vectors support", RankVectorsFieldMapper.FEATURE_FLAG.isEnabled()); - } - private final ElementType elementType; private final int dims; @@ -66,6 +62,11 @@ public RankVectorsFieldMapperTests() { this.dims = ElementType.BIT == elementType ? 4 * Byte.SIZE : 4; } + @Override + protected Collection getPlugins() { + return Collections.singletonList(new LocalStateRankVectors(SETTINGS)); + } + @Override protected void minimalMapping(XContentBuilder b) throws IOException { indexMapping(b, IndexVersion.current()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldTypeTests.java b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldTypeTests.java similarity index 67% rename from server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldTypeTests.java rename to x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldTypeTests.java index b4cbbc4730d7c..59c5b0414910d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldTypeTests.java +++ b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldTypeTests.java @@ -1,20 +1,21 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.index.mapper.vectors; +package org.elasticsearch.xpack.rank.vectors.mapper; -import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.mapper.FieldTypeTestCase; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.vectors.RankVectorsFieldMapper.RankVectorsFieldType; -import org.junit.BeforeClass; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; +import org.elasticsearch.license.License; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.license.internal.XPackLicenseStatus; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.xpack.rank.vectors.mapper.RankVectorsFieldMapper.RankVectorsFieldType; import java.io.IOException; import java.util.Collections; @@ -25,23 +26,17 @@ public class RankVectorsFieldTypeTests extends FieldTypeTestCase { - @BeforeClass - public static void setup() { - assumeTrue("Requires rank-vectors support", RankVectorsFieldMapper.FEATURE_FLAG.isEnabled()); - } + private final XPackLicenseState licenseState = new XPackLicenseState( + System::currentTimeMillis, + new XPackLicenseStatus(License.OperationMode.TRIAL, true, null) + ); private RankVectorsFieldType createFloatFieldType() { - return new RankVectorsFieldType( - "f", - DenseVectorFieldMapper.ElementType.FLOAT, - BBQ_MIN_DIMS, - IndexVersion.current(), - Collections.emptyMap() - ); + return new RankVectorsFieldType("f", DenseVectorFieldMapper.ElementType.FLOAT, BBQ_MIN_DIMS, licenseState, Collections.emptyMap()); } - private RankVectorsFieldType createByteFieldType() { - return new RankVectorsFieldType("f", DenseVectorFieldMapper.ElementType.BYTE, 5, IndexVersion.current(), Collections.emptyMap()); + private RankVectorsFieldMapper.RankVectorsFieldType createByteFieldType() { + return new RankVectorsFieldType("f", DenseVectorFieldMapper.ElementType.BYTE, 5, licenseState, Collections.emptyMap()); } public void testHasDocValues() { @@ -84,9 +79,9 @@ public void testFielddataBuilder() { public void testDocValueFormat() { RankVectorsFieldType fft = createFloatFieldType(); - expectThrows(IllegalArgumentException.class, () -> fft.docValueFormat(null, null)); + assertEquals(DocValueFormat.DENSE_VECTOR, fft.docValueFormat(null, null)); RankVectorsFieldType bft = createByteFieldType(); - expectThrows(IllegalArgumentException.class, () -> bft.docValueFormat(null, null)); + assertEquals(DocValueFormat.DENSE_VECTOR, bft.docValueFormat(null, null)); } public void testFetchSourceValue() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsScriptDocValuesTests.java b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsScriptDocValuesTests.java similarity index 95% rename from server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsScriptDocValuesTests.java rename to x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsScriptDocValuesTests.java index c38ed0f60f0ae..f0b00849557bd 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsScriptDocValuesTests.java +++ b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsScriptDocValuesTests.java @@ -1,24 +1,22 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.index.mapper.vectors; +package org.elasticsearch.xpack.rank.vectors.mapper; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.util.BytesRef; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType; +import org.elasticsearch.index.mapper.vectors.RankVectorsScriptDocValues; import org.elasticsearch.script.field.vectors.ByteRankVectorsDocValuesField; import org.elasticsearch.script.field.vectors.FloatRankVectorsDocValuesField; import org.elasticsearch.script.field.vectors.RankVectors; import org.elasticsearch.script.field.vectors.RankVectorsDocValuesField; import org.elasticsearch.test.ESTestCase; -import org.junit.BeforeClass; import java.io.IOException; import java.nio.ByteBuffer; @@ -29,11 +27,6 @@ public class RankVectorsScriptDocValuesTests extends ESTestCase { - @BeforeClass - public static void setup() { - assumeTrue("Requires rank-vectors support", RankVectorsFieldMapper.FEATURE_FLAG.isEnabled()); - } - public void testFloatGetVectorValueAndGetMagnitude() throws IOException { int dims = 3; float[][][] vectors = { { { 1, 1, 1 }, { 1, 1, 2 }, { 1, 1, 3 } }, { { 1, 0, 2 } } }; diff --git a/server/src/test/java/org/elasticsearch/script/RankVectorsScoreScriptUtilsTests.java b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsScoreScriptUtilsTests.java similarity index 93% rename from server/src/test/java/org/elasticsearch/script/RankVectorsScoreScriptUtilsTests.java rename to x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsScoreScriptUtilsTests.java index 917cc2069a293..da0340a22c074 100644 --- a/server/src/test/java/org/elasticsearch/script/RankVectorsScoreScriptUtilsTests.java +++ b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsScoreScriptUtilsTests.java @@ -1,26 +1,23 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.script; +package org.elasticsearch.xpack.rank.vectors.script; import org.apache.lucene.util.VectorUtil; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType; -import org.elasticsearch.index.mapper.vectors.RankVectorsFieldMapper; -import org.elasticsearch.index.mapper.vectors.RankVectorsScriptDocValuesTests; -import org.elasticsearch.script.RankVectorsScoreScriptUtils.MaxSimDotProduct; -import org.elasticsearch.script.RankVectorsScoreScriptUtils.MaxSimInvHamming; +import org.elasticsearch.script.ScoreScript; import org.elasticsearch.script.field.vectors.BitRankVectorsDocValuesField; import org.elasticsearch.script.field.vectors.ByteRankVectorsDocValuesField; import org.elasticsearch.script.field.vectors.FloatRankVectorsDocValuesField; import org.elasticsearch.script.field.vectors.RankVectorsDocValuesField; import org.elasticsearch.test.ESTestCase; -import org.junit.BeforeClass; +import org.elasticsearch.xpack.rank.vectors.mapper.RankVectorsScriptDocValuesTests; +import org.elasticsearch.xpack.rank.vectors.script.RankVectorsScoreScriptUtils.MaxSimDotProduct; +import org.elasticsearch.xpack.rank.vectors.script.RankVectorsScoreScriptUtils.MaxSimInvHamming; import java.io.IOException; import java.util.Arrays; @@ -33,11 +30,6 @@ public class RankVectorsScoreScriptUtilsTests extends ESTestCase { - @BeforeClass - public static void setup() { - assumeTrue("Requires rank-vectors support", RankVectorsFieldMapper.FEATURE_FLAG.isEnabled()); - } - public void testFloatMultiVectorClassBindings() throws IOException { String fieldName = "vector"; int dims = 5; @@ -88,7 +80,7 @@ public void testFloatMultiVectorClassBindings() throws IOException { // Check each function rejects query vectors with the wrong dimension IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> new RankVectorsScoreScriptUtils.MaxSimDotProduct(scoreScript, invalidQueryVector, fieldName) + () -> new MaxSimDotProduct(scoreScript, invalidQueryVector, fieldName) ); assertThat( e.getMessage(), @@ -336,7 +328,4 @@ public void testByteBoundaries() throws IOException { } } - public void testDimMismatch() throws IOException { - - } } diff --git a/server/src/test/java/org/elasticsearch/script/field/vectors/RankVectorsTests.java b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsTests.java similarity index 80% rename from server/src/test/java/org/elasticsearch/script/field/vectors/RankVectorsTests.java rename to x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsTests.java index ca7608b10aed9..570e2a59926aa 100644 --- a/server/src/test/java/org/elasticsearch/script/field/vectors/RankVectorsTests.java +++ b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsTests.java @@ -1,19 +1,19 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.script.field.vectors; +package org.elasticsearch.xpack.rank.vectors.script; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.VectorUtil; -import org.elasticsearch.index.mapper.vectors.RankVectorsFieldMapper; +import org.elasticsearch.script.field.vectors.ByteRankVectors; +import org.elasticsearch.script.field.vectors.FloatRankVectors; +import org.elasticsearch.script.field.vectors.RankVectors; +import org.elasticsearch.script.field.vectors.VectorIterator; import org.elasticsearch.test.ESTestCase; -import org.junit.BeforeClass; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -21,11 +21,6 @@ public class RankVectorsTests extends ESTestCase { - @BeforeClass - public static void setup() { - assumeTrue("Requires rank-vectors support", RankVectorsFieldMapper.FEATURE_FLAG.isEnabled()); - } - public void testByteUnsupported() { int count = randomIntBetween(1, 16); int dims = randomIntBetween(1, 16); diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/30_rank_vectors.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors.yml similarity index 92% rename from rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/30_rank_vectors.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors.yml index ecf34f46c3383..791712ee925a5 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/30_rank_vectors.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors.yml @@ -1,11 +1,7 @@ setup: - requires: - capabilities: - - method: POST - path: /_search - capabilities: [ rank_vectors_field_mapper ] - test_runner_features: capabilities - reason: "Support for rank vectors field mapper capability required" + cluster_features: [ "rank_vectors" ] + reason: "requires rank_vectors feature" --- "Test create multi-vector field": - do: diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/181_rank_vectors_dv_fields_api.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors_dv_fields_api.yml similarity index 71% rename from modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/181_rank_vectors_dv_fields_api.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors_dv_fields_api.yml index f37e554fca7bf..bcfb9bcd79b90 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/181_rank_vectors_dv_fields_api.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors_dv_fields_api.yml @@ -1,11 +1,7 @@ setup: - requires: - capabilities: - - method: POST - path: /_search - capabilities: [ rank_vectors_script_access ] - test_runner_features: capabilities - reason: "Support for rank vector field script access capability required" + cluster_features: [ "rank_vectors" ] + reason: "requires rank_vectors feature" - skip: features: headers @@ -17,6 +13,8 @@ setup: number_of_shards: 1 mappings: properties: + id: + type: keyword vector: type: rank_vectors dims: 5 @@ -33,6 +31,7 @@ setup: index: test-index id: "1" body: + id: "1" vector: [[230.0, 300.33, -34.8988, 15.555, -200.0], [-0.5, 100.0, -13, 14.8, -156.0]] byte_vector: [[8, 5, -15, 1, -7], [-1, 115, -3, 4, -128]] bit_vector: [[8, 5, -15, 1, -7], [-1, 115, -3, 4, -128]] @@ -42,6 +41,7 @@ setup: index: test-index id: "3" body: + id: "3" vector: [[0.5, 111.3, -13.0, 14.8, -156.0]] byte_vector: [[2, 18, -5, 0, -124]] bit_vector: [[2, 18, -5, 0, -124]] @@ -176,3 +176,35 @@ setup: - match: {hits.hits.1._id: "3"} - close_to: {hits.hits.1._score: {value: 2, error: 0.01}} +--- +"Test doc value vector access": + - skip: + features: close_to + + - do: + search: + _source: false + index: test-index + body: + docvalue_fields: [name, bit_vector, byte_vector, vector] + sort: [{id: "asc"}] + + - match: {hits.hits.0._id: "1"} + # vector: [[230.0, 300.33, -34.8988, 15.555, -200.0], [-0.5, 100.0, -13, 14.8, -156.0]] + - close_to: {hits.hits.0.fields.vector.0.0.0: {value: 230, error: 0.0001}} + - close_to: {hits.hits.0.fields.vector.0.0.1: {value: 300.33, error: 0.0001}} + - close_to: {hits.hits.0.fields.vector.0.0.2: {value: -34.8988, error: 0.0001}} + - close_to: {hits.hits.0.fields.vector.0.0.3: {value: 15.555, error: 0.0001}} + - close_to: {hits.hits.0.fields.vector.0.0.4: {value: -200, error: 0.0001}} + - close_to: {hits.hits.0.fields.vector.0.1.0: {value: -0.5, error: 0.0001}} + - close_to: {hits.hits.0.fields.vector.0.1.1: {value: 100, error: 0.0001}} + - close_to: {hits.hits.0.fields.vector.0.1.2: {value: -13, error: 0.0001}} + - close_to: {hits.hits.0.fields.vector.0.1.3: {value: 14.8, error: 0.0001}} + - close_to: {hits.hits.0.fields.vector.0.1.4: {value: -156, error: 0.0001}} + + # byte_vector: [[8, 5, -15, 1, -7], [-1, 115, -3, 4, -128]] + - match: {hits.hits.0.fields.byte_vector.0.0: [8, 5, -15, 1, -7]} + - match: {hits.hits.0.fields.bit_vector.0.0: [8, 5, -15, 1, -7]} + - match: {hits.hits.0.fields.byte_vector.0.1: [-1, 115, -3, 4, -128]} + - match: {hits.hits.0.fields.bit_vector.0.1: [-1, 115, -3, 4, -128]} + diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/141_rank_vectors_max_sim.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors_max_sim.yml similarity index 95% rename from modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/141_rank_vectors_max_sim.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors_max_sim.yml index 7c46fbc9a26a5..acaf1b99b626e 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/141_rank_vectors_max_sim.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors_max_sim.yml @@ -1,11 +1,7 @@ setup: - requires: - capabilities: - - method: POST - path: /_search - capabilities: [ rank_vectors_script_max_sim_with_bugfix ] - test_runner_features: capabilities - reason: "Support for rank vectors max-sim functions capability required" + cluster_features: [ "rank_vectors" ] + reason: "requires rank_vectors feature" - skip: features: headers