From cee7520161d5cda987af56535757dcee63ffaa99 Mon Sep 17 00:00:00 2001 From: faberf Date: Tue, 20 Aug 2024 15:52:47 +0200 Subject: [PATCH 1/4] implemented retriever --- .../classification/ImageClassification.kt | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassification.kt b/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassification.kt index 5b8f5d036..fb3ecc4b8 100644 --- a/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassification.kt +++ b/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassification.kt @@ -4,11 +4,14 @@ import org.vitrivr.engine.base.features.external.api.AbstractApi import org.vitrivr.engine.base.features.external.common.ExternalFesAnalyser import org.vitrivr.engine.core.context.IndexContext import org.vitrivr.engine.core.context.QueryContext +import org.vitrivr.engine.core.features.AbstractRetriever +import org.vitrivr.engine.core.model.content.element.ContentElement import org.vitrivr.engine.core.model.content.element.ImageContent import org.vitrivr.engine.core.model.descriptor.struct.LabelDescriptor import org.vitrivr.engine.core.model.metamodel.Analyser.Companion.merge import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.query.Query +import org.vitrivr.engine.core.model.query.bool.SimpleBooleanQuery import org.vitrivr.engine.core.model.retrievable.Retrievable import org.vitrivr.engine.core.model.types.Value import org.vitrivr.engine.core.operators.Operator @@ -62,11 +65,9 @@ class ImageClassification : ExternalFesAnalyser() */ override fun newExtractor(field: Schema.Field, input: Operator, context: IndexContext) = ImageClassificationExtractor(input, field, this, merge(field, context)) - override fun newRetrieverForContent(field: Schema.Field, content: Collection, context: QueryContext): Retriever { - TODO("Not yet implemented") - } - override fun newRetrieverForQuery(field: Schema.Field, query: Query, context: QueryContext): Retriever { - TODO("Not yet implemented") + require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } + require(query is SimpleBooleanQuery<*>) { "The query is not a boolean query. This is a programmer's error!" } + return object : AbstractRetriever(field, query, context){} } } \ No newline at end of file From 4883d0e6bc9436015bee06800014416c5cf18be5 Mon Sep 17 00:00:00 2001 From: faberf Date: Fri, 23 Aug 2024 13:29:27 +0200 Subject: [PATCH 2/4] implemented retrieverfromcontent --- .../classification/ImageClassification.kt | 26 +++++++++++++++---- .../ImageClassificationExtractor.kt | 9 ++++--- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassification.kt b/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassification.kt index fb3ecc4b8..d4f953ab5 100644 --- a/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassification.kt +++ b/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassification.kt @@ -5,8 +5,10 @@ import org.vitrivr.engine.base.features.external.common.ExternalFesAnalyser import org.vitrivr.engine.core.context.IndexContext import org.vitrivr.engine.core.context.QueryContext import org.vitrivr.engine.core.features.AbstractRetriever +import org.vitrivr.engine.core.model.content.Content import org.vitrivr.engine.core.model.content.element.ContentElement import org.vitrivr.engine.core.model.content.element.ImageContent +import org.vitrivr.engine.core.model.content.element.TextContent import org.vitrivr.engine.core.model.descriptor.struct.LabelDescriptor import org.vitrivr.engine.core.model.metamodel.Analyser.Companion.merge import org.vitrivr.engine.core.model.metamodel.Schema @@ -25,14 +27,14 @@ import java.util.* * @author Fynn Faber * @version 1.0.0 */ -class ImageClassification : ExternalFesAnalyser() { +class ImageClassification : ExternalFesAnalyser, LabelDescriptor>() { companion object{ const val CLASSES_PARAMETER_NAME = "classes" const val THRESHOLD_PARAMETER_NAME = "threshold" const val TOPK_PARAMETER_NAME = "top_k" } - override val contentClasses = setOf(ImageContent::class) + override val contentClasses = setOf(ImageContent::class, TextContent::class) override val descriptorClass = LabelDescriptor::class /** @@ -63,11 +65,25 @@ class ImageClassification : ExternalFesAnalyser() * @param context The [IndexContext] to use with the [ImageClassification]. * @return [ImageClassification] */ - override fun newExtractor(field: Schema.Field, input: Operator, context: IndexContext) = ImageClassificationExtractor(input, field, this, merge(field, context)) + override fun newExtractor(field: Schema.Field, LabelDescriptor>, input: Operator, context: IndexContext) = ImageClassificationExtractor(input, field, this, merge(field, context)) - override fun newRetrieverForQuery(field: Schema.Field, query: Query, context: QueryContext): Retriever { + override fun newRetrieverForQuery(field: Schema.Field, LabelDescriptor>, query: Query, context: QueryContext): Retriever, LabelDescriptor> { require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } require(query is SimpleBooleanQuery<*>) { "The query is not a boolean query. This is a programmer's error!" } - return object : AbstractRetriever(field, query, context){} + return object : AbstractRetriever, LabelDescriptor>(field, query, context){} + } + + override fun newRetrieverForContent( + field: Schema.Field, LabelDescriptor>, + content: Collection>, + context: QueryContext + ): Retriever, LabelDescriptor> { + val firstContent = content.first() + if (content.size != 1 || firstContent !is TextContent) { + throw IllegalArgumentException("The content does not match the expected type. This is a programmer's error!") + } + val query = SimpleBooleanQuery(value = Value.String(firstContent.content), attributeName = "label") + return newRetrieverForQuery(field, query, context) + } } \ No newline at end of file diff --git a/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassificationExtractor.kt b/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassificationExtractor.kt index 9684ef944..0e5057377 100644 --- a/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassificationExtractor.kt +++ b/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassificationExtractor.kt @@ -6,6 +6,7 @@ import org.vitrivr.engine.base.features.external.common.FesExtractor import org.vitrivr.engine.base.features.external.implementations.classification.ImageClassification.Companion.CLASSES_PARAMETER_NAME import org.vitrivr.engine.base.features.external.implementations.classification.ImageClassification.Companion.THRESHOLD_PARAMETER_NAME import org.vitrivr.engine.base.features.external.implementations.classification.ImageClassification.Companion.TOPK_PARAMETER_NAME +import org.vitrivr.engine.core.model.content.element.ContentElement import org.vitrivr.engine.core.model.content.element.ImageContent import org.vitrivr.engine.core.model.descriptor.struct.LabelDescriptor import org.vitrivr.engine.core.model.metamodel.Schema @@ -21,10 +22,10 @@ import java.util.* */ class ImageClassificationExtractor( input: Operator, - field: Schema.Field?, - analyser: ExternalFesAnalyser, + field: Schema.Field, LabelDescriptor>?, + analyser: ExternalFesAnalyser, LabelDescriptor>, parameters: Map -) : FesExtractor(input, field, analyser, parameters) { +) : FesExtractor, LabelDescriptor>(input, field, analyser, parameters) { /** The [ZeroShotClassificationApi] used to perform extraction with. */ @@ -48,7 +49,7 @@ class ImageClassificationExtractor( val flatResults = this.api.analyseBatched( retrievables.flatMap { - this.filterContent(it).map { it to classes } + this.filterContent(it).filterIsInstance().map { it to classes } }).mapIndexed { idx, result -> result.mapIndexed { idy, confidence -> LabelDescriptor( From 0e2c7ded255c861aedc21523eff051f837c417c2 Mon Sep 17 00:00:00 2001 From: faberf Date: Fri, 23 Aug 2024 13:48:20 +0200 Subject: [PATCH 3/4] fixed issue with classifier --- .../classification/ImageClassificationExtractor.kt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassificationExtractor.kt b/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassificationExtractor.kt index 0e5057377..44155f7f0 100644 --- a/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassificationExtractor.kt +++ b/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassificationExtractor.kt @@ -47,10 +47,11 @@ class ImageClassificationExtractor( val topK = this.parameters[TOPK_PARAMETER_NAME]?.toInt() ?: 1 val threshold = this.parameters[THRESHOLD_PARAMETER_NAME]?.toFloat() ?: 0.0f - val flatResults = this.api.analyseBatched( - retrievables.flatMap { - this.filterContent(it).filterIsInstance().map { it to classes } - }).mapIndexed { idx, result -> + val content = retrievables.mapIndexed { idx, retrievable -> + this.filterContent(retrievable).filterIsInstance().map { idx to (it to classes) } + }.flatten() + + return this.api.analyseBatched(content.map{it.second}).zip(content.map{it.first}).map { (result, idx) -> result.mapIndexed { idy, confidence -> LabelDescriptor( UUID.randomUUID(), @@ -63,6 +64,5 @@ class ImageClassificationExtractor( ) }.filter { it.confidence.value >= threshold }.sortedByDescending { it.confidence.value }.take(topK) } - return flatResults } } \ No newline at end of file From 11901cb2c5f95198c2e59a4ffd1b5441ab207c6c Mon Sep 17 00:00:00 2001 From: Ralph Gasser Date: Fri, 23 Aug 2024 15:13:30 +0200 Subject: [PATCH 4/4] Minor adjustment plus added missing implementation. Signed-off-by: Ralph Gasser --- .../classification/ImageClassification.kt | 45 ++++++++++++++++--- 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassification.kt b/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassification.kt index d4f953ab5..70f0fac3e 100644 --- a/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassification.kt +++ b/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassification.kt @@ -10,10 +10,13 @@ import org.vitrivr.engine.core.model.content.element.ContentElement import org.vitrivr.engine.core.model.content.element.ImageContent import org.vitrivr.engine.core.model.content.element.TextContent import org.vitrivr.engine.core.model.descriptor.struct.LabelDescriptor +import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor +import org.vitrivr.engine.core.model.metamodel.Analyser import org.vitrivr.engine.core.model.metamodel.Analyser.Companion.merge import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.query.Query import org.vitrivr.engine.core.model.query.bool.SimpleBooleanQuery +import org.vitrivr.engine.core.model.query.proximity.ProximityQuery import org.vitrivr.engine.core.model.retrievable.Retrievable import org.vitrivr.engine.core.model.types.Value import org.vitrivr.engine.core.operators.Operator @@ -67,23 +70,53 @@ class ImageClassification : ExternalFesAnalyser, LabelDescript */ override fun newExtractor(field: Schema.Field, LabelDescriptor>, input: Operator, context: IndexContext) = ImageClassificationExtractor(input, field, this, merge(field, context)) + /** + * Generates and returns a new [Retriever] instance for this [ImageClassification]. + * + * @param field The [Schema.Field] to create an [Retriever] for. + * @param query The [Query] to use with the [Retriever]. + * @param context The [QueryContext] to use with the [Retriever]. + * + * @return A new [Retriever] instance for this [Analyser] + */ override fun newRetrieverForQuery(field: Schema.Field, LabelDescriptor>, query: Query, context: QueryContext): Retriever, LabelDescriptor> { require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } require(query is SimpleBooleanQuery<*>) { "The query is not a boolean query. This is a programmer's error!" } return object : AbstractRetriever, LabelDescriptor>(field, query, context){} } + /** + * Generates and returns a new [Retriever] instance for this [ImageClassification]. + * + * Invoking this method involves converting the provided [FloatVectorDescriptor] into a [ProximityQuery] that can be used to retrieve similar [ImageContent] elements. + * + * @param field The [Schema.Field] to create an [Retriever] for. + * @param descriptors An array of [FloatVectorDescriptor] elements to use with the [Retriever] + * @param context The [QueryContext] to use with the [Retriever] + */ + override fun newRetrieverForDescriptors(field: Schema.Field, LabelDescriptor>, descriptors: Collection, context: QueryContext): Retriever, LabelDescriptor> { + val descriptor = descriptors.firstOrNull()?.label ?: throw IllegalArgumentException("No label descriptor provided.") + val query = SimpleBooleanQuery(value = descriptor, attributeName = "label") + return newRetrieverForQuery(field, query, context) + } + + /** + * Generates and returns a new [Retriever] instance for this [ImageClassification]. + * + * Invoking this method involves converting the provided [ImageContent] and the [QueryContext] into a [FloatVectorDescriptor] + * that can be used to retrieve similar [ImageContent] elements. + * + * @param field The [Schema.Field] to create an [Retriever] for. + * @param content An array of [Content] elements to use with the [Retriever] + * @param context The [QueryContext] to use with the [Retriever] + */ override fun newRetrieverForContent( field: Schema.Field, LabelDescriptor>, content: Collection>, context: QueryContext ): Retriever, LabelDescriptor> { - val firstContent = content.first() - if (content.size != 1 || firstContent !is TextContent) { - throw IllegalArgumentException("The content does not match the expected type. This is a programmer's error!") - } - val query = SimpleBooleanQuery(value = Value.String(firstContent.content), attributeName = "label") + val first = content.filterIsInstance().firstOrNull() ?: throw IllegalArgumentException("The content does not contain any text.") + val query = SimpleBooleanQuery(value = Value.String(first.content), attributeName = "label") return newRetrieverForQuery(field, query, context) - } } \ No newline at end of file