From 2edf6508a91cf351ed91b2628c396e71013663fb Mon Sep 17 00:00:00 2001 From: Ralph Gasser Date: Wed, 21 Aug 2024 09:25:17 +0200 Subject: [PATCH] Adds HueHistogram feature adapted from Cineast. Signed-off-by: Ralph Gasser --- .../features/averagecolor/AverageColor.kt | 6 +- .../engine/core/model/color/ColorConverter.kt | 44 +++++ .../core/model/color/hsv/HSVColorContainer.kt | 26 +++ .../MutableRGBFloatColorContainer.kt | 2 +- .../color/{ => rgb}/RGBByteColorContainer.kt | 4 +- .../color/{ => rgb}/RGBFloatColorContainer.kt | 2 +- .../image/AverageImageContentAggregator.kt | 4 +- .../RepresentativeImageContentAggregator.kt | 4 +- .../feature/huehistogram/HueHistogram.kt | 151 ++++++++++++++++++ .../huehistogram/HueHistogramExtractor.kt | 45 ++++++ .../huehistogram/HueHistogramRetriever.kt | 42 +++++ ...trivr.engine.core.model.metamodel.Analyser | 1 + .../external/api/ImageCaptioningApi.kt | 2 +- 13 files changed, 322 insertions(+), 11 deletions(-) create mode 100644 vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/ColorConverter.kt create mode 100644 vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/hsv/HSVColorContainer.kt rename vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/{ => rgb}/MutableRGBFloatColorContainer.kt (98%) rename vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/{ => rgb}/RGBByteColorContainer.kt (86%) rename vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/{ => rgb}/RGBFloatColorContainer.kt (98%) create mode 100644 vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogram.kt create mode 100644 vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogramExtractor.kt create mode 100644 vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogramRetriever.kt diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColor.kt index 2fcf01f2e..4b0f52c8d 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColor.kt @@ -2,9 +2,9 @@ package org.vitrivr.engine.core.features.averagecolor import org.vitrivr.engine.core.context.IndexContext import org.vitrivr.engine.core.context.QueryContext -import org.vitrivr.engine.core.model.color.MutableRGBFloatColorContainer -import org.vitrivr.engine.core.model.color.RGBByteColorContainer -import org.vitrivr.engine.core.model.color.RGBFloatColorContainer +import org.vitrivr.engine.core.model.color.rgb.MutableRGBFloatColorContainer +import org.vitrivr.engine.core.model.color.rgb.RGBByteColorContainer +import org.vitrivr.engine.core.model.color.rgb.RGBFloatColorContainer import org.vitrivr.engine.core.model.content.Content import org.vitrivr.engine.core.model.content.element.ImageContent import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/ColorConverter.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/ColorConverter.kt new file mode 100644 index 000000000..871019e69 --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/ColorConverter.kt @@ -0,0 +1,44 @@ +package org.vitrivr.engine.core.model.color + +import org.vitrivr.engine.core.model.color.hsv.HSVColorContainer +import org.vitrivr.engine.core.model.color.rgb.RGBByteColorContainer +import org.vitrivr.engine.core.model.color.rgb.RGBFloatColorContainer + +/** + * A color converter that provides methods to convert between different color spaces. + * + * Adapted from Cineast. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +object ColorConverter { + + /** + * Converts a [RGBByteColorContainer] to a [HSVColorContainer]. + * + * @param rgb The [RGBByteColorContainer] to convert. + * @return [HSVColorContainer] + */ + fun rgbToHsv(rgb: RGBByteColorContainer): HSVColorContainer = rgbToHsv(rgb.toFloatContainer()) + + /** + * Converts a [RGBFloatColorContainer] to a [HSVColorContainer]. + * + * @param rgb The [RGBFloatColorContainer] to convert. + * @return [HSVColorContainer] + */ + fun rgbToHsv(rgb: RGBFloatColorContainer): HSVColorContainer { + val max = maxOf(rgb.red, rgb.green, rgb.blue) + val min = minOf(rgb.red, rgb.green, rgb.blue) + val d = max - min + val s = if (max == 0f) 0f else d / max + val h = when { + d == 0f -> 0f + max == rgb.red -> (rgb.green - rgb.blue) / d + (if (rgb.green < rgb.blue) 6 else 0) + max == rgb.green -> (rgb.blue - rgb.red) / d + 2 + else -> (rgb.red - rgb.green) / d + 4 + } + return HSVColorContainer(h, s, max) + } +} \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/hsv/HSVColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/hsv/HSVColorContainer.kt new file mode 100644 index 000000000..00b5f0761 --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/hsv/HSVColorContainer.kt @@ -0,0 +1,26 @@ +package org.vitrivr.engine.core.model.color.hsv + +/** + * A container for HSV colors. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +@JvmInline +value class HSVColorContainer private constructor(private val colors: FloatArray) { + + constructor(hue: Float, saturation: Float, value: Float) : this(floatArrayOf(hue, saturation, value)) + constructor(hue: Double, saturation: Double, value: Double) : this(floatArrayOf(hue.toFloat(), saturation.toFloat(), value.toFloat())) + + /** Accessor for the hue component of the [HSVColorContainer]. */ + val hue: Float + get() = this.colors[0] + + /** Accessor for the saturation component of the [HSVColorContainer]. */ + val saturation: Float + get() = this.colors[1] + + /** Accessor for the value component of the [HSVColorContainer]. */ + val value: Float + get() = this.colors[2] +} \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/MutableRGBFloatColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/rgb/MutableRGBFloatColorContainer.kt similarity index 98% rename from vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/MutableRGBFloatColorContainer.kt rename to vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/rgb/MutableRGBFloatColorContainer.kt index 26a2f3721..aa5eb78a2 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/MutableRGBFloatColorContainer.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/rgb/MutableRGBFloatColorContainer.kt @@ -1,4 +1,4 @@ -package org.vitrivr.engine.core.model.color +package org.vitrivr.engine.core.model.color.rgb import kotlin.math.max import kotlin.math.min diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBByteColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/rgb/RGBByteColorContainer.kt similarity index 86% rename from vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBByteColorContainer.kt rename to vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/rgb/RGBByteColorContainer.kt index 8db668c17..dfe77f15e 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBByteColorContainer.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/rgb/RGBByteColorContainer.kt @@ -1,4 +1,4 @@ -package org.vitrivr.engine.core.model.color +package org.vitrivr.engine.core.model.color.rgb /** * A container for RGB colors. @@ -31,7 +31,9 @@ value class RGBByteColorContainer(private val rgb: Int) { get() = (this.rgb and 0xFF).toUByte() /** + * Converts this [RGBByteColorContainer] to an [RGBFloatColorContainer]. * + * @return [RGBFloatColorContainer] representation of this [RGBByteColorContainer]. */ fun toFloatContainer(): RGBFloatColorContainer = RGBFloatColorContainer(this.red.toFloat() / 255f, this.green.toFloat() / 255f, this.blue.toFloat() / 255f) diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBFloatColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/rgb/RGBFloatColorContainer.kt similarity index 98% rename from vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBFloatColorContainer.kt rename to vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/rgb/RGBFloatColorContainer.kt index 3218bc69e..a478f6086 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBFloatColorContainer.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/rgb/RGBFloatColorContainer.kt @@ -1,4 +1,4 @@ -package org.vitrivr.engine.core.model.color +package org.vitrivr.engine.core.model.color.rgb import org.vitrivr.engine.core.model.types.Value import kotlin.math.max diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/AverageImageContentAggregator.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/AverageImageContentAggregator.kt index 397cb4855..c857e3bda 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/AverageImageContentAggregator.kt +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/AverageImageContentAggregator.kt @@ -2,8 +2,8 @@ package org.vitrivr.engine.index.aggregators.image import org.vitrivr.engine.core.context.Context import org.vitrivr.engine.core.context.IndexContext -import org.vitrivr.engine.core.model.color.MutableRGBFloatColorContainer -import org.vitrivr.engine.core.model.color.RGBByteColorContainer +import org.vitrivr.engine.core.model.color.rgb.MutableRGBFloatColorContainer +import org.vitrivr.engine.core.model.color.rgb.RGBByteColorContainer import org.vitrivr.engine.core.model.content.decorators.SourcedContent import org.vitrivr.engine.core.model.content.decorators.TemporalContent import org.vitrivr.engine.core.model.content.element.ContentElement diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/RepresentativeImageContentAggregator.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/RepresentativeImageContentAggregator.kt index 51ddbec61..e38d944da 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/RepresentativeImageContentAggregator.kt +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/RepresentativeImageContentAggregator.kt @@ -2,8 +2,8 @@ package org.vitrivr.engine.index.aggregators.image import org.vitrivr.engine.core.context.Context import org.vitrivr.engine.core.context.IndexContext -import org.vitrivr.engine.core.model.color.MutableRGBFloatColorContainer -import org.vitrivr.engine.core.model.color.RGBByteColorContainer +import org.vitrivr.engine.core.model.color.rgb.MutableRGBFloatColorContainer +import org.vitrivr.engine.core.model.color.rgb.RGBByteColorContainer import org.vitrivr.engine.core.model.content.element.ContentElement import org.vitrivr.engine.core.model.content.element.ImageContent import org.vitrivr.engine.core.model.retrievable.Ingested diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogram.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogram.kt new file mode 100644 index 000000000..609cfe14b --- /dev/null +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogram.kt @@ -0,0 +1,151 @@ +package org.vitrivr.engine.module.features.feature.huehistogram + +import org.vitrivr.engine.core.context.IndexContext +import org.vitrivr.engine.core.context.QueryContext +import org.vitrivr.engine.core.model.color.ColorConverter +import org.vitrivr.engine.core.model.color.hsv.HSVColorContainer +import org.vitrivr.engine.core.model.color.rgb.RGBByteColorContainer +import org.vitrivr.engine.core.model.content.Content +import org.vitrivr.engine.core.model.content.element.ContentElement +import org.vitrivr.engine.core.model.content.element.ImageContent +import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor +import org.vitrivr.engine.core.model.metamodel.Analyser +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.query.Query +import org.vitrivr.engine.core.model.query.proximity.ProximityQuery +import org.vitrivr.engine.core.model.retrievable.Retrievable +import org.vitrivr.engine.core.model.types.Value +import org.vitrivr.engine.core.operators.Operator +import org.vitrivr.engine.core.operators.ingest.Extractor +import org.vitrivr.engine.core.operators.retrieve.Retriever +import java.util.* +import kotlin.reflect.KClass + +/** + * A hue histogram [Analyser] for [ImageContent] objects. + * + * Migrated from Cineast. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class HueHistogram : Analyser { + + companion object { + private const val VECTOR_SIZE = 16 + } + + override val contentClasses: Set>> = setOf(ImageContent::class) + override val descriptorClass: KClass = FloatVectorDescriptor::class + + /** + * Generates a prototypical [FloatVectorDescriptor] for this [HueHistogram]. + * + * @param field [Schema.Field] to create the prototype for. + * @return [FloatVectorDescriptor] + */ + override fun prototype(field: Schema.Field<*, *>): FloatVectorDescriptor = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.FloatVector(VECTOR_SIZE)) + + /** + * Generates and returns a new [HueHistogramExtractor] instance for this [HueHistogram]. + * + * @param name The name of the [HueHistogramExtractor]. + * @param input The [Operator] that acts as input to the new [Extractor]. + * @param context The [IndexContext] to use with the [Extractor]. + * + * @return A new [Extractor] instance for this [Analyser] + * @throws [UnsupportedOperationException], if this [Analyser] does not support the creation of an [Extractor] instance. + */ + override fun newExtractor(name: String, input: Operator, context: IndexContext) = HueHistogramExtractor(input, this, null) + + /** + * Generates and returns a new [HueHistogramExtractor] instance for this [HueHistogram]. + * + * @param field The [Schema.Field] to create an [Extractor] for. + * @param input The [Operator] that acts as input to the new [Extractor]. + * @param context The [IndexContext] to use with the [Extractor]. + * + * @return A new [Extractor] instance for this [Analyser] + * @throws [UnsupportedOperationException], if this [Analyser] does not support the creation of an [Extractor] instance. + */ + override fun newExtractor(field: Schema.Field, input: Operator, context: IndexContext) = HueHistogramExtractor(input, this, field) + + /** + * Generates and returns a new [HueHistogramRetriever] instance for this [HueHistogram]. + * + * @param field The [Schema.Field] to create an [Retriever] for. + * @param query The [Query] to use with the [Retriever]. + * @param context The [QueryContext] to use with the [Retriever]. + * + * @return A new [HueHistogramRetriever] instance for this [HueHistogram] + */ + override fun newRetrieverForQuery(field: Schema.Field, query: Query, context: QueryContext): HueHistogramRetriever { + require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } + require(query is ProximityQuery<*> && query.value is Value.FloatVector) { "The query is not a ProximityQuery." } + @Suppress("UNCHECKED_CAST") + return HueHistogramRetriever(field, query as ProximityQuery, context) + } + + /** + * Generates and returns a new [HueHistogramRetriever] instance for this [HueHistogram]. + * + * Invoking this method involves converting the provided [FloatVectorDescriptor] into a [ProximityQuery] that can be used to retrieve similar [ImageContent] elements. + * + * @param field The [Schema.Field] to create an [Retriever] for. + * @param descriptors An array of [FloatVectorDescriptor] elements to use with the [Retriever] + * @param context The [QueryContext] to use with the [Retriever] + */ + override fun newRetrieverForDescriptors(field: Schema.Field, descriptors: Collection, context: QueryContext): HueHistogramRetriever { + require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } + + /* Prepare query parameters. */ + val k = context.getProperty(field.fieldName, "limit")?.toLongOrNull() ?: 1000L + val fetchVector = context.getProperty(field.fieldName, "returnDescriptor")?.toBooleanStrictOrNull() ?: false + + /* Return retriever. */ + return this.newRetrieverForQuery(field, ProximityQuery(value = descriptors.first().vector, k = k, fetchVector = fetchVector), context) + } + + /** + * Generates and returns a new [HueHistogramRetriever] instance for this [HueHistogram]. + * + * Invoking this method involves converting the provided [ImageContent] and the [QueryContext] into a [FloatVectorDescriptor] + * that can be used to retrieve similar [ImageContent] elements. + * + * @param field The [Schema.Field] to create an [Retriever] for. + * @param content An array of [Content] elements to use with the [Retriever] + * @param context The [QueryContext] to use with the [Retriever] + */ + override fun newRetrieverForContent(field: Schema.Field, content: Collection, context: QueryContext) = this.newRetrieverForDescriptors(field, content.map { this.analyse(it) }, context) + + /** + * Performs the [HueHistogram] analysis on the provided [ImageContent] and returns a [FloatVectorDescriptor] that represents the result. + * + * @param content [ImageContent] to be analysed. + * @return [FloatVectorDescriptor] result of the analysis. + */ + fun analyse(content: ImageContent): FloatVectorDescriptor { + /* Generate histogram. */ + val hist = FloatArray(VECTOR_SIZE) + val colors = content.content.getRGB(0, 0, content.content.width, content.content.height, null, 0, content.content.width) + for (color in colors) { + val container: HSVColorContainer = ColorConverter.rgbToHsv(RGBByteColorContainer(color)) + if (container.saturation > 0.2f && container.value > 0.3f) { + val h: Float = container.hue * hist.size + val idx = h.toInt() + hist[idx] += h - idx + hist[(idx + 1) % hist.size] += idx + 1 - h + } + } + + /* Normalize hist. */ + val sum = hist.sum() + if (sum > 1f) { + for (i in hist.indices) { + hist[i] /= sum + } + } + + return FloatVectorDescriptor(vector = Value.FloatVector(hist)) + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogramExtractor.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogramExtractor.kt new file mode 100644 index 000000000..f16974cb5 --- /dev/null +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogramExtractor.kt @@ -0,0 +1,45 @@ +package org.vitrivr.engine.module.features.feature.huehistogram + +import org.vitrivr.engine.core.features.AbstractExtractor +import org.vitrivr.engine.core.features.metadata.source.file.FileSourceMetadataExtractor +import org.vitrivr.engine.core.model.content.ContentType +import org.vitrivr.engine.core.model.content.element.ImageContent +import org.vitrivr.engine.core.model.descriptor.Descriptor +import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.retrievable.Retrievable +import org.vitrivr.engine.core.operators.Operator +import org.vitrivr.engine.core.operators.ingest.Extractor +import org.vitrivr.engine.core.source.file.FileSource +import org.vitrivr.engine.module.features.feature.ehd.EHD + +/** + * An [Extractor] implementation for the [HueHistogram] analyser. + * + * @see [EHD] + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class HueHistogramExtractor(input: Operator, analyser: HueHistogram, field: Schema.Field?) : AbstractExtractor(input, analyser, field) { + /** + * Internal method to check, if [Retrievable] matches this [Extractor] and should thus be processed. + * + * [FileSourceMetadataExtractor] implementation only works with [Retrievable] that contain a [FileSource]. + * + * @param retrievable The [Retrievable] to check. + * @return True on match, false otherwise, + */ + override fun matches(retrievable: Retrievable): Boolean = retrievable.content.any { it.type == ContentType.BITMAP_IMAGE } + + /** + * Internal method to perform extraction on [Retrievable]. + * + * @param retrievable The [Retrievable] to process. + * @return List of resulting [Descriptor]s. + */ + override fun extract(retrievable: Retrievable): List { + val content = retrievable.content.filterIsInstance() + return content.map { (this.analyser as HueHistogram).analyse(it).copy(retrievableId = retrievable.id, field = this.field) } + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogramRetriever.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogramRetriever.kt new file mode 100644 index 000000000..0edbd3264 --- /dev/null +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogramRetriever.kt @@ -0,0 +1,42 @@ +package org.vitrivr.engine.module.features.feature.huehistogram + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.flow.flow +import org.vitrivr.engine.core.context.QueryContext +import org.vitrivr.engine.core.features.AbstractRetriever +import org.vitrivr.engine.core.math.correspondence.LinearCorrespondence +import org.vitrivr.engine.core.model.content.element.ImageContent +import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.query.proximity.ProximityQuery +import org.vitrivr.engine.core.model.retrievable.attributes.DistanceAttribute +import org.vitrivr.engine.core.model.types.Value +import org.vitrivr.engine.core.operators.retrieve.Retriever + +/** + * [Retriever] implementation for the [HueHistogram] analyser. + * + * @see [HueHistogram] + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class HueHistogramRetriever(field: Schema.Field, query: ProximityQuery, context: QueryContext) : AbstractRetriever(field, query, context) { + + companion object { + /** [LinearCorrespondence] for [HueHistogramRetriever]. Maximum distance is taken from Cineast implementation. */ + private val CORRESPONDENCE = LinearCorrespondence(16f) + } + + override fun toFlow(scope: CoroutineScope) = flow { + this@HueHistogramRetriever.reader.queryAndJoin(this@HueHistogramRetriever.query).forEach { + val distance = it.filteredAttribute() + if (distance != null) { + it.addAttribute(CORRESPONDENCE(distance)) + } else { + this@HueHistogramRetriever.logger.warn { "No distance attribute found for descriptor ${it.id}." } + } + emit(it) + } + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser b/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser index 446faf6f8..032bde4d2 100644 --- a/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser +++ b/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser @@ -1,3 +1,4 @@ org.vitrivr.engine.module.features.feature.ehd.EHD +org.vitrivr.engine.module.features.feature.huehistogram.HueHistogram org.vitrivr.engine.module.features.feature.external.implementations.dino.DINO org.vitrivr.engine.module.features.feature.external.implementations.clip.CLIP \ No newline at end of file diff --git a/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/api/ImageCaptioningApi.kt b/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/api/ImageCaptioningApi.kt index a2602c5e6..a3a073e73 100644 --- a/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/api/ImageCaptioningApi.kt +++ b/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/api/ImageCaptioningApi.kt @@ -82,7 +82,7 @@ class ImageCaptioningApi(host: String, model: String, timeoutMs: Long, pollingIn */ override suspend fun pollBatchedJob(jobId: String): JobResult> = try { this.imageCaptioningApi.getBatchedJobResultsApiTasksImageCaptioningBatchedJobsJobGet(jobId).body().let { result -> - val values = result.result?.map { Value.Text(it.caption.trim() ?: "") } + val values = result.result?.map { Value.Text(it.caption.trim()) } JobResult(result.status, values) } } catch (e: Throwable) {