From cee7520161d5cda987af56535757dcee63ffaa99 Mon Sep 17 00:00:00 2001 From: faberf Date: Tue, 20 Aug 2024 15:52:47 +0200 Subject: [PATCH 01/20] implemented retriever --- .../classification/ImageClassification.kt | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassification.kt b/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassification.kt index 5b8f5d036..fb3ecc4b8 100644 --- a/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassification.kt +++ b/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassification.kt @@ -4,11 +4,14 @@ import org.vitrivr.engine.base.features.external.api.AbstractApi import org.vitrivr.engine.base.features.external.common.ExternalFesAnalyser import org.vitrivr.engine.core.context.IndexContext import org.vitrivr.engine.core.context.QueryContext +import org.vitrivr.engine.core.features.AbstractRetriever +import org.vitrivr.engine.core.model.content.element.ContentElement import org.vitrivr.engine.core.model.content.element.ImageContent import org.vitrivr.engine.core.model.descriptor.struct.LabelDescriptor import org.vitrivr.engine.core.model.metamodel.Analyser.Companion.merge import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.query.Query +import org.vitrivr.engine.core.model.query.bool.SimpleBooleanQuery import org.vitrivr.engine.core.model.retrievable.Retrievable import org.vitrivr.engine.core.model.types.Value import org.vitrivr.engine.core.operators.Operator @@ -62,11 +65,9 @@ class ImageClassification : ExternalFesAnalyser() */ override fun newExtractor(field: Schema.Field, input: Operator, context: IndexContext) = ImageClassificationExtractor(input, field, this, merge(field, context)) - override fun newRetrieverForContent(field: Schema.Field, content: Collection, context: QueryContext): Retriever { - TODO("Not yet implemented") - } - override fun newRetrieverForQuery(field: Schema.Field, query: Query, context: QueryContext): Retriever { - TODO("Not yet implemented") + require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } + require(query is SimpleBooleanQuery<*>) { "The query is not a boolean query. This is a programmer's error!" } + return object : AbstractRetriever(field, query, context){} } } \ No newline at end of file From c27192456d074fa6c2998eb94ea6a80ca7b23cae Mon Sep 17 00:00:00 2001 From: Ralph Gasser Date: Wed, 21 Aug 2024 08:17:00 +0200 Subject: [PATCH 02/20] Correspondence functions now have a dedicated interface. Signed-off-by: Ralph Gasser --- .../engine/core/features/AbstractRetriever.kt | 7 +++- .../features/averagecolor/AverageColor.kt | 39 +++++++++---------- .../averagecolor/AverageColorRetriever.kt | 34 +++++++--------- .../correspondence/CorrespondenceFunction.kt | 20 ++++++++++ .../correspondence/LinearCorrespondence.kt | 14 +++++++ 5 files changed, 72 insertions(+), 42 deletions(-) create mode 100644 vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/math/correspondence/CorrespondenceFunction.kt create mode 100644 vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/math/correspondence/LinearCorrespondence.kt diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/AbstractRetriever.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/AbstractRetriever.kt index 859523f55..6d7324d1a 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/AbstractRetriever.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/AbstractRetriever.kt @@ -1,5 +1,7 @@ package org.vitrivr.engine.core.features +import io.github.oshai.kotlinlogging.KLogger +import io.github.oshai.kotlinlogging.KotlinLogging import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.flow @@ -20,13 +22,16 @@ import org.vitrivr.engine.core.operators.retrieve.Retriever * @see [AbstractRetriever] * * @author Rahel Arnold - * @version 1.0.0 + * @version 1.0.1 */ abstract class AbstractRetriever, D : Descriptor<*>>(override val field: Schema.Field, val query: Query, val context: QueryContext) : Retriever { /** The [DescriptorReader] instance used by this [AbstractRetriever]. */ protected val reader: DescriptorReader by lazy { this.field.getReader() } + /** The [KLogger] instance used by this [AbstractRetriever]. */ + protected val logger: KLogger = KotlinLogging.logger {} + /** * Simplest implementation of the retrieval logic simply hand the [Query] to the [DescriptorReader] and emit the results. * diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColor.kt index dfe802759..2fcf01f2e 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColor.kt @@ -30,28 +30,9 @@ import java.util.* * @version 1.0.0 */ class AverageColor : Analyser { - override val contentClasses = setOf(ImageContent::class) override val descriptorClass = FloatVectorDescriptor::class - companion object { - /** - * Performs the [AverageColor] analysis on the provided [List] of [ImageContent] elements. - * - * @param content The [List] of [ImageContent] elements. - * @return [List] of [FloatVectorDescriptor]s. - */ - fun analyse(content: Collection): List = content.map { - val color = MutableRGBFloatColorContainer() - val rgb = it.content.getRGBArray() - rgb.forEach { c -> color += RGBByteColorContainer(c) } - - /* Generate descriptor. */ - val averageColor = RGBFloatColorContainer(color.red / rgb.size, color.green / rgb.size, color.blue / rgb.size) - FloatVectorDescriptor(UUID.randomUUID(), null, averageColor.toVector()) - } - } - /** * Generates a prototypical [FloatVectorDescriptor] for this [AverageColor]. * @@ -97,7 +78,7 @@ class AverageColor : Analyser { require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } require(query is ProximityQuery<*> && query.value is Value.FloatVector) { "The query is not a ProximityQuery." } @Suppress("UNCHECKED_CAST") - return AverageColorRetriever(field, query as ProximityQuery) + return AverageColorRetriever(field, query as ProximityQuery, context) } /** @@ -131,5 +112,21 @@ class AverageColor : Analyser { * @param context The [QueryContext] to use with the [Retriever] */ override fun newRetrieverForContent(field: Schema.Field, content: Collection, context: QueryContext): AverageColorRetriever = - this.newRetrieverForDescriptors(field, analyse(content), context) + this.newRetrieverForDescriptors(field, this.analyse(content), context) + + /** + * Performs the [AverageColor] analysis on the provided [List] of [ImageContent] elements. + * + * @param content The [List] of [ImageContent] elements. + * @return [List] of [FloatVectorDescriptor]s. + */ + fun analyse(content: Collection): List = content.map { + val color = MutableRGBFloatColorContainer() + val rgb = it.content.getRGBArray() + rgb.forEach { c -> color += RGBByteColorContainer(c) } + + /* Generate descriptor. */ + val averageColor = RGBFloatColorContainer(color.red / rgb.size, color.green / rgb.size, color.blue / rgb.size) + FloatVectorDescriptor(UUID.randomUUID(), null, averageColor.toVector()) + } } diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColorRetriever.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColorRetriever.kt index 48a788b23..126640442 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColorRetriever.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColorRetriever.kt @@ -1,16 +1,15 @@ package org.vitrivr.engine.core.features.averagecolor -import io.github.oshai.kotlinlogging.KLogger -import io.github.oshai.kotlinlogging.KotlinLogging import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.flow.flow +import org.vitrivr.engine.core.context.QueryContext +import org.vitrivr.engine.core.features.AbstractRetriever +import org.vitrivr.engine.core.math.correspondence.LinearCorrespondence import org.vitrivr.engine.core.model.content.element.ImageContent import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.query.proximity.ProximityQuery -import org.vitrivr.engine.core.model.retrievable.Retrieved import org.vitrivr.engine.core.model.retrievable.attributes.DistanceAttribute -import org.vitrivr.engine.core.model.retrievable.attributes.ScoreAttribute import org.vitrivr.engine.core.model.types.Value import org.vitrivr.engine.core.operators.retrieve.Retriever @@ -20,28 +19,23 @@ import org.vitrivr.engine.core.operators.retrieve.Retriever * @see [AverageColor] * * @author Luca Rossetto - * @version 1.1.0 + * @version 1.2.0 */ -class AverageColorRetriever( - override val field: Schema.Field, - private val query: ProximityQuery -) : Retriever { - - private val logger: KLogger = KotlinLogging.logger {} +class AverageColorRetriever(field: Schema.Field, query: ProximityQuery, context: QueryContext) : AbstractRetriever(field, query, context) { companion object { - private const val MAXIMUM_DISTANCE = 3f - fun scoringFunction(retrieved: Retrieved): Float { - val distance = retrieved.filteredAttribute()?.distance ?: return 0.0f - return 1.0f - (distance / MAXIMUM_DISTANCE) - } + /** [LinearCorrespondence] for [AverageColorRetriever]. */ + private val CORRESPONDENCE = LinearCorrespondence(3f) } override fun toFlow(scope: CoroutineScope) = flow { - val reader = this@AverageColorRetriever.field.getReader() - logger.debug { "Flow init with query $query" } - reader.queryAndJoin(this@AverageColorRetriever.query).forEach { - it.addAttribute(ScoreAttribute.Similarity(scoringFunction(it))) + this@AverageColorRetriever.reader.queryAndJoin(this@AverageColorRetriever.query).forEach { + val distance = it.filteredAttribute() + if (distance != null) { + it.addAttribute(CORRESPONDENCE(distance)) + } else { + this@AverageColorRetriever.logger.warn { "No distance attribute found for descriptor ${it.id}." } + } emit(it) } } diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/math/correspondence/CorrespondenceFunction.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/math/correspondence/CorrespondenceFunction.kt new file mode 100644 index 000000000..9a4555529 --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/math/correspondence/CorrespondenceFunction.kt @@ -0,0 +1,20 @@ +package org.vitrivr.engine.core.math.correspondence + +import org.vitrivr.engine.core.model.retrievable.attributes.DistanceAttribute +import org.vitrivr.engine.core.model.retrievable.attributes.ScoreAttribute + +/** + * A [CorrespondenceFunction] is used to compute the [ScoreAttribute.Similarity] for a given [DistanceAttribute]. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +interface CorrespondenceFunction { + /** + * Computes the score for a given [DistanceAttribute]. + * + * @param distance [DistanceAttribute] for which to compute the score. + * @return [ScoreAttribute.Similarity] for the given [DistanceAttribute]. + */ + operator fun invoke(distance: DistanceAttribute): ScoreAttribute.Similarity +} \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/math/correspondence/LinearCorrespondence.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/math/correspondence/LinearCorrespondence.kt new file mode 100644 index 000000000..7a0b2a7da --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/math/correspondence/LinearCorrespondence.kt @@ -0,0 +1,14 @@ +package org.vitrivr.engine.core.math.correspondence + +import org.vitrivr.engine.core.model.retrievable.attributes.DistanceAttribute +import org.vitrivr.engine.core.model.retrievable.attributes.ScoreAttribute + +/** + * A linear [CorrespondenceFunction] that is based on a maximum distance. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class LinearCorrespondence(private val maximumDistance: Float) : CorrespondenceFunction { + override fun invoke(distance: DistanceAttribute): ScoreAttribute.Similarity = ScoreAttribute.Similarity(1.0f - (distance.distance / this.maximumDistance)) +} \ No newline at end of file From 7abc435b4ac11e2ff0309a2d4859b26505967a4b Mon Sep 17 00:00:00 2001 From: Ralph Gasser Date: Wed, 21 Aug 2024 08:20:10 +0200 Subject: [PATCH 03/20] Adds EHD analyser implementation. Signed-off-by: Ralph Gasser --- gradle.properties | 1 + .../engine/core/util/math/MathHelper.kt | 8 +- vitrivr-engine-module-features/build.gradle | 3 + .../engine/module/features/feature/ehd/EHD.kt | 198 ++++++++++++++++++ .../features/feature/ehd/EHDExtractor.kt | 44 ++++ .../features/feature/ehd/EHDRetriever.kt | 42 ++++ ...trivr.engine.core.model.metamodel.Analyser | 30 +-- 7 files changed, 297 insertions(+), 29 deletions(-) create mode 100644 vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/ehd/EHD.kt create mode 100644 vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/ehd/EHDExtractor.kt create mode 100644 vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/ehd/EHDRetriever.kt diff --git a/gradle.properties b/gradle.properties index cf611c897..90b5f8205 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,4 @@ +version_boofcv=1.1.5 version_caffeine=3.1.8 version_clikt=4.2.0 version_commonsmath3=3.6.1 diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/util/math/MathHelper.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/util/math/MathHelper.kt index db6e57a54..07b1c5bea 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/util/math/MathHelper.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/util/math/MathHelper.kt @@ -8,10 +8,16 @@ import kotlin.math.sqrt * A collection of helper functions for common, mathematical operations. * * @author Ralph Gasser - * @version 1.0.0 + * @version 1.0.1 */ object MathHelper { + /** Square root of 2. */ + val SQRT2: Double = sqrt(2.0) + + /** Square root of 2 as a [Float]. */ + val SQRT2_f: Float = SQRT2.toFloat() + /** * Normalizes a [Array] of [Value.Float] with respect to the L2 (euclidian) norm. * The method will return a new array and leave the original array unchanged. diff --git a/vitrivr-engine-module-features/build.gradle b/vitrivr-engine-module-features/build.gradle index e693c8bdb..f1a9e10db 100644 --- a/vitrivr-engine-module-features/build.gradle +++ b/vitrivr-engine-module-features/build.gradle @@ -9,6 +9,9 @@ repositories { dependencies { api project(':vitrivr-engine-core') + + /* BoofCV for image modification and processing. */ + implementation group: 'org.boofcv', name: 'boofcv-io', version: version_boofcv } /* Publication of vitrivr engine query to Maven Central. */ diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/ehd/EHD.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/ehd/EHD.kt new file mode 100644 index 000000000..70c0a6178 --- /dev/null +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/ehd/EHD.kt @@ -0,0 +1,198 @@ +package org.vitrivr.engine.module.features.feature.ehd + +import boofcv.io.image.ConvertBufferedImage +import boofcv.struct.image.GrayU8 +import org.vitrivr.engine.core.context.IndexContext +import org.vitrivr.engine.core.context.QueryContext +import org.vitrivr.engine.core.model.content.Content +import org.vitrivr.engine.core.model.content.element.ContentElement +import org.vitrivr.engine.core.model.content.element.ImageContent +import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor +import org.vitrivr.engine.core.model.metamodel.Analyser +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.query.Query +import org.vitrivr.engine.core.model.query.proximity.ProximityQuery +import org.vitrivr.engine.core.model.retrievable.Retrievable +import org.vitrivr.engine.core.model.types.Value +import org.vitrivr.engine.core.operators.Operator +import org.vitrivr.engine.core.operators.ingest.Extractor +import org.vitrivr.engine.core.operators.retrieve.Retriever +import org.vitrivr.engine.core.util.math.MathHelper +import java.util.* +import kotlin.reflect.KClass + +/** + * A MPEG 7 Edge Histogram Descriptor (EHD) [Analyser] for [ImageContent] objects. + * + * Migrated from Cineast. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class EHD : Analyser { + + companion object { + private const val VECTOR_SIZE = 80 + private val MV: FloatArray = floatArrayOf(1f, -1f, 1f, -1f) + private val MH: FloatArray = floatArrayOf(1f, 1f, -1f, -1f) + private val M45: FloatArray = floatArrayOf(MathHelper.SQRT2_f, 0f, 0f, -MathHelper.SQRT2_f) + private val M135: FloatArray = floatArrayOf(0f, MathHelper.SQRT2_f, -MathHelper.SQRT2_f, 0f) + private val MN: FloatArray = floatArrayOf(2f, -2f, -2f, 2f) + + /** + * + */ + private fun edgeType(i1: Int, i2: Int, i3: Int, i4: Int): Int { + val coeffs = floatArrayOf( + MV[0] * i1 + MV[1] * i2 + MV[2] * i3 + MV[3] * i4, + MH[0] * i1 + MH[1] * i2 + MH[2] * i3 + MH[3] * i4, + M45[0] * i1 + M45[1] * i2 + M45[2] * i3 + M45[3] * i4, + M135[0] * i1 + M135[1] * i2 + M135[2] * i3 + M135[3] * i4, + MN[0] * i1 + MN[1] * i2 + MN[2] * i3 + MN[3] * i4, + ) + + var maxid = 0 + for (i in 1..4) { + if (coeffs[maxid] < coeffs[i]) { + maxid = i + } + } + + if (coeffs[maxid] >= 14) { + return maxid + } + + return -1 + } + } + + override val contentClasses: Set>> = setOf(ImageContent::class) + override val descriptorClass: KClass = FloatVectorDescriptor::class + + /** + * Generates a prototypical [FloatVectorDescriptor] for this [EHD]. + * + * @param field [Schema.Field] to create the prototype for. + * @return [FloatVectorDescriptor] + */ + override fun prototype(field: Schema.Field<*, *>): FloatVectorDescriptor = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.FloatVector(VECTOR_SIZE)) + + /** + * Generates and returns a new [EHDExtractor] instance for this [EHD]. + * + * @param name The name of the [EHDExtractor]. + * @param input The [Operator] that acts as input to the new [Extractor]. + * @param context The [IndexContext] to use with the [Extractor]. + * + * @return A new [Extractor] instance for this [Analyser] + * @throws [UnsupportedOperationException], if this [Analyser] does not support the creation of an [Extractor] instance. + */ + override fun newExtractor(name: String, input: Operator, context: IndexContext) = EHDExtractor(input, this, null) + + /** + * Generates and returns a new [EHDExtractor] instance for this [EHD]. + * + * @param field The [Schema.Field] to create an [Extractor] for. + * @param input The [Operator] that acts as input to the new [Extractor]. + * @param context The [IndexContext] to use with the [Extractor]. + * + * @return A new [Extractor] instance for this [Analyser] + * @throws [UnsupportedOperationException], if this [Analyser] does not support the creation of an [Extractor] instance. + */ + override fun newExtractor(field: Schema.Field, input: Operator, context: IndexContext) = EHDExtractor(input, this, field) + + /** + * Generates and returns a new [EHDRetriever] instance for this [EHD]. + * + * @param field The [Schema.Field] to create an [Retriever] for. + * @param query The [Query] to use with the [Retriever]. + * @param context The [QueryContext] to use with the [Retriever]. + * + * @return A new [EHDRetriever] instance for this [EHD] + */ + override fun newRetrieverForQuery(field: Schema.Field, query: Query, context: QueryContext): EHDRetriever { + require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } + require(query is ProximityQuery<*> && query.value is Value.FloatVector) { "The query is not a ProximityQuery." } + @Suppress("UNCHECKED_CAST") + return EHDRetriever(field, query as ProximityQuery, context) + } + + /** + * Generates and returns a new [EHDRetriever] instance for this [EHD]. + * + * Invoking this method involves converting the provided [FloatVectorDescriptor] into a [ProximityQuery] that can be used to retrieve similar [ImageContent] elements. + * + * @param field The [Schema.Field] to create an [Retriever] for. + * @param descriptors An array of [FloatVectorDescriptor] elements to use with the [Retriever] + * @param context The [QueryContext] to use with the [Retriever] + */ + override fun newRetrieverForDescriptors(field: Schema.Field, descriptors: Collection, context: QueryContext): EHDRetriever { + require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } + + /* Prepare query parameters. */ + val k = context.getProperty(field.fieldName, "limit")?.toLongOrNull() ?: 1000L + val fetchVector = context.getProperty(field.fieldName, "returnDescriptor")?.toBooleanStrictOrNull() ?: false + + /* Return retriever. */ + return this.newRetrieverForQuery(field, ProximityQuery(value = descriptors.first().vector, k = k, fetchVector = fetchVector), context) + } + + /** + * Generates and returns a new [EHDRetriever] instance for this [EHD]. + * + * Invoking this method involves converting the provided [ImageContent] and the [QueryContext] into a [FloatVectorDescriptor] + * that can be used to retrieve similar [ImageContent] elements. + * + * @param field The [Schema.Field] to create an [Retriever] for. + * @param content An array of [Content] elements to use with the [Retriever] + * @param context The [QueryContext] to use with the [Retriever] + */ + override fun newRetrieverForContent(field: Schema.Field, content: Collection, context: QueryContext): EHDRetriever = + this.newRetrieverForDescriptors(field, content.map { this.analyse(it) }, context) + + /** + * Performs the [EHD] analysis on the provided [ImageContent] and returns a [FloatVectorDescriptor] that represents the result. + * + * @param content [ImageContent] to be analysed. + * @return [FloatVectorDescriptor] result of the analysis. + */ + fun analyse(content: ImageContent): FloatVectorDescriptor { + val gray: GrayU8 = ConvertBufferedImage.convertFrom(content.content, null as GrayU8?) + val width: Int = content.content.width + val height: Int = content.content.height + val hist = FloatArray(VECTOR_SIZE) + for (x in 0..3) { + for (y in 0..3) { + val subImage: GrayU8 = gray.subimage( + width * x / 4, height * y / 4, width * (x + 1) / 4, height * (y + 1) / 4, + null + ) + var count = 0 + val tmp = IntArray(5) + var xx = 0 + while (xx < subImage.getWidth() - 1) { + var yy = 0 + while (yy < subImage.getHeight() - 1) { + count++ + val index: Int = edgeType( + subImage.unsafe_get(xx, yy), + subImage.unsafe_get(xx + 1, yy), + subImage.unsafe_get(xx, yy + 1), + subImage.unsafe_get(xx + 1, yy + 1) + ) + if (index > -1) { + tmp[index]++ + } + yy += 2 + } + xx += 2 + } + val offset = (4 * x + y) * 5 + for (i in 0..4) { + hist[offset + i] += (tmp[i].toFloat()) / count.toFloat() + } + } + } + return FloatVectorDescriptor(vector = Value.FloatVector(hist)) + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/ehd/EHDExtractor.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/ehd/EHDExtractor.kt new file mode 100644 index 000000000..3de500929 --- /dev/null +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/ehd/EHDExtractor.kt @@ -0,0 +1,44 @@ +package org.vitrivr.engine.module.features.feature.ehd + +import org.vitrivr.engine.core.features.AbstractExtractor +import org.vitrivr.engine.core.features.metadata.source.file.FileSourceMetadataExtractor +import org.vitrivr.engine.core.model.content.ContentType +import org.vitrivr.engine.core.model.content.element.ImageContent +import org.vitrivr.engine.core.model.descriptor.Descriptor +import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.retrievable.Retrievable +import org.vitrivr.engine.core.operators.Operator +import org.vitrivr.engine.core.operators.ingest.Extractor +import org.vitrivr.engine.core.source.file.FileSource + +/** + * [Extractor] implementation for the [EHD] analyser. + * + * @see [EHD] + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class EHDExtractor(input: Operator, analyser: EHD, field: Schema.Field?) : AbstractExtractor(input, analyser, field) { + /** + * Internal method to check, if [Retrievable] matches this [Extractor] and should thus be processed. + * + * [FileSourceMetadataExtractor] implementation only works with [Retrievable] that contain a [FileSource]. + * + * @param retrievable The [Retrievable] to check. + * @return True on match, false otherwise, + */ + override fun matches(retrievable: Retrievable): Boolean = retrievable.content.any { it.type == ContentType.BITMAP_IMAGE } + + /** + * Internal method to perform extraction on [Retrievable]. + * + * @param retrievable The [Retrievable] to process. + * @return List of resulting [Descriptor]s. + */ + override fun extract(retrievable: Retrievable): List { + val content = retrievable.content.filterIsInstance() + return content.map { (this.analyser as EHD).analyse(it).copy(retrievableId = retrievable.id, field = this.field) } + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/ehd/EHDRetriever.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/ehd/EHDRetriever.kt new file mode 100644 index 000000000..30ee064ce --- /dev/null +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/ehd/EHDRetriever.kt @@ -0,0 +1,42 @@ +package org.vitrivr.engine.module.features.feature.ehd + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.flow.flow +import org.vitrivr.engine.core.context.QueryContext +import org.vitrivr.engine.core.features.AbstractRetriever +import org.vitrivr.engine.core.math.correspondence.LinearCorrespondence +import org.vitrivr.engine.core.model.content.element.ImageContent +import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.query.proximity.ProximityQuery +import org.vitrivr.engine.core.model.retrievable.attributes.DistanceAttribute +import org.vitrivr.engine.core.model.types.Value +import org.vitrivr.engine.core.operators.retrieve.Retriever + +/** + * [Retriever] implementation for the [EHD] analyser. + * + * @see [EHD] + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class EHDRetriever(field: Schema.Field, query: ProximityQuery, context: QueryContext) : AbstractRetriever(field, query, context) { + + companion object { + /** [LinearCorrespondence] for [EHDRetriever]. Maximum distance is taken from Cineast implementation. */ + private val CORRESPONDENCE = LinearCorrespondence(16 / 4f) + } + + override fun toFlow(scope: CoroutineScope) = flow { + this@EHDRetriever.reader.queryAndJoin(this@EHDRetriever.query).forEach { + val distance = it.filteredAttribute() + if (distance != null) { + it.addAttribute(CORRESPONDENCE(distance)) + } else { + this@EHDRetriever.logger.warn { "No distance attribute found for descriptor ${it.id}." } + } + emit(it) + } + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser b/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser index a8c5e73b6..446faf6f8 100644 --- a/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser +++ b/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser @@ -1,29 +1,3 @@ +org.vitrivr.engine.module.features.feature.ehd.EHD org.vitrivr.engine.module.features.feature.external.implementations.dino.DINO -org.vitrivr.engine.module.features.feature.external.implementations.clip.CLIP -org.vitrivr.engine.module.features.feature.migration.WhisperASR -org.vitrivr.engine.module.features.feature.migration.ProvidedOCR -org.vitrivr.engine.module.features.feature.migration.AudioTranscription -org.vitrivr.engine.module.features.feature.skeleton.SkeletonPose -org.vitrivr.engine.module.features.feature.migration.MedianColor -org.vitrivr.engine.module.features.feature.migration.CLD -org.vitrivr.engine.module.features.feature.migration.CLDReduced15 -org.vitrivr.engine.module.features.feature.migration.AverageFuzzyHistNormalized -org.vitrivr.engine.module.features.feature.migration.AverageFuzzyHist -org.vitrivr.engine.module.features.feature.migration.MedianFuzzyHist -org.vitrivr.engine.module.features.feature.migration.HueHistogram -org.vitrivr.engine.module.features.feature.migration.DominantEdgeGrid -org.vitrivr.engine.module.features.feature.migration.EHD -org.vitrivr.engine.module.features.feature.migration.AverageColorGrid8 -org.vitrivr.engine.module.features.feature.migration.AverageColorGrid8Reduced15 -org.vitrivr.engine.module.features.feature.migration.EdgeGrid16 -org.vitrivr.engine.module.features.feature.migration.DominantEdgeGrid16 -org.vitrivr.engine.module.features.feature.migration.VisualTextCoEmbedding -org.vitrivr.engine.module.features.feature.migration.OpenCLIP -org.vitrivr.engine.module.features.feature.migration.HOGMF25k512 -org.vitrivr.engine.module.features.feature.migration.SURFMF25K512 -org.vitrivr.engine.module.features.feature.migration.InceptionResNetV2 -org.vitrivr.engine.module.features.feature.migration.ConceptMasksADE20k -org.vitrivr.engine.module.features.feature.averagecolorraster.AverageColorRaster -org.vitrivr.engine.module.features.feature.migration.EdgeARP88 -org.vitrivr.engine.module.features.feature.migration.DominantColor -org.vitrivr.engine.module.features.feature.migration.DominantEdgeGrid8 \ No newline at end of file +org.vitrivr.engine.module.features.feature.external.implementations.clip.CLIP \ No newline at end of file From d1c29c3b8a95b18d9694f3e6965a0e766ca83da7 Mon Sep 17 00:00:00 2001 From: Ralph Gasser Date: Wed, 21 Aug 2024 08:50:07 +0200 Subject: [PATCH 04/20] Fixes compilation error. Signed-off-by: Ralph Gasser --- .../engine/core/features/averagecolor/AverageColorExtractor.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColorExtractor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColorExtractor.kt index 3ecf15dff..b580a3012 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColorExtractor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColorExtractor.kt @@ -39,6 +39,6 @@ class AverageColorExtractor(input: Operator, analyser: AverageColor */ override fun extract(retrievable: Retrievable): List { val content = retrievable.content.filterIsInstance() - return AverageColor.analyse(content).map { it.copy(retrievableId = retrievable.id, field = this@AverageColorExtractor.field) } + return (this.analyser as AverageColor).analyse(content).map { it.copy(retrievableId = retrievable.id, field = this@AverageColorExtractor.field) } } } \ No newline at end of file From 2edf6508a91cf351ed91b2628c396e71013663fb Mon Sep 17 00:00:00 2001 From: Ralph Gasser Date: Wed, 21 Aug 2024 09:25:17 +0200 Subject: [PATCH 05/20] Adds HueHistogram feature adapted from Cineast. Signed-off-by: Ralph Gasser --- .../features/averagecolor/AverageColor.kt | 6 +- .../engine/core/model/color/ColorConverter.kt | 44 +++++ .../core/model/color/hsv/HSVColorContainer.kt | 26 +++ .../MutableRGBFloatColorContainer.kt | 2 +- .../color/{ => rgb}/RGBByteColorContainer.kt | 4 +- .../color/{ => rgb}/RGBFloatColorContainer.kt | 2 +- .../image/AverageImageContentAggregator.kt | 4 +- .../RepresentativeImageContentAggregator.kt | 4 +- .../feature/huehistogram/HueHistogram.kt | 151 ++++++++++++++++++ .../huehistogram/HueHistogramExtractor.kt | 45 ++++++ .../huehistogram/HueHistogramRetriever.kt | 42 +++++ ...trivr.engine.core.model.metamodel.Analyser | 1 + .../external/api/ImageCaptioningApi.kt | 2 +- 13 files changed, 322 insertions(+), 11 deletions(-) create mode 100644 vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/ColorConverter.kt create mode 100644 vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/hsv/HSVColorContainer.kt rename vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/{ => rgb}/MutableRGBFloatColorContainer.kt (98%) rename vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/{ => rgb}/RGBByteColorContainer.kt (86%) rename vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/{ => rgb}/RGBFloatColorContainer.kt (98%) create mode 100644 vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogram.kt create mode 100644 vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogramExtractor.kt create mode 100644 vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogramRetriever.kt diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColor.kt index 2fcf01f2e..4b0f52c8d 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColor.kt @@ -2,9 +2,9 @@ package org.vitrivr.engine.core.features.averagecolor import org.vitrivr.engine.core.context.IndexContext import org.vitrivr.engine.core.context.QueryContext -import org.vitrivr.engine.core.model.color.MutableRGBFloatColorContainer -import org.vitrivr.engine.core.model.color.RGBByteColorContainer -import org.vitrivr.engine.core.model.color.RGBFloatColorContainer +import org.vitrivr.engine.core.model.color.rgb.MutableRGBFloatColorContainer +import org.vitrivr.engine.core.model.color.rgb.RGBByteColorContainer +import org.vitrivr.engine.core.model.color.rgb.RGBFloatColorContainer import org.vitrivr.engine.core.model.content.Content import org.vitrivr.engine.core.model.content.element.ImageContent import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/ColorConverter.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/ColorConverter.kt new file mode 100644 index 000000000..871019e69 --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/ColorConverter.kt @@ -0,0 +1,44 @@ +package org.vitrivr.engine.core.model.color + +import org.vitrivr.engine.core.model.color.hsv.HSVColorContainer +import org.vitrivr.engine.core.model.color.rgb.RGBByteColorContainer +import org.vitrivr.engine.core.model.color.rgb.RGBFloatColorContainer + +/** + * A color converter that provides methods to convert between different color spaces. + * + * Adapted from Cineast. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +object ColorConverter { + + /** + * Converts a [RGBByteColorContainer] to a [HSVColorContainer]. + * + * @param rgb The [RGBByteColorContainer] to convert. + * @return [HSVColorContainer] + */ + fun rgbToHsv(rgb: RGBByteColorContainer): HSVColorContainer = rgbToHsv(rgb.toFloatContainer()) + + /** + * Converts a [RGBFloatColorContainer] to a [HSVColorContainer]. + * + * @param rgb The [RGBFloatColorContainer] to convert. + * @return [HSVColorContainer] + */ + fun rgbToHsv(rgb: RGBFloatColorContainer): HSVColorContainer { + val max = maxOf(rgb.red, rgb.green, rgb.blue) + val min = minOf(rgb.red, rgb.green, rgb.blue) + val d = max - min + val s = if (max == 0f) 0f else d / max + val h = when { + d == 0f -> 0f + max == rgb.red -> (rgb.green - rgb.blue) / d + (if (rgb.green < rgb.blue) 6 else 0) + max == rgb.green -> (rgb.blue - rgb.red) / d + 2 + else -> (rgb.red - rgb.green) / d + 4 + } + return HSVColorContainer(h, s, max) + } +} \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/hsv/HSVColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/hsv/HSVColorContainer.kt new file mode 100644 index 000000000..00b5f0761 --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/hsv/HSVColorContainer.kt @@ -0,0 +1,26 @@ +package org.vitrivr.engine.core.model.color.hsv + +/** + * A container for HSV colors. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +@JvmInline +value class HSVColorContainer private constructor(private val colors: FloatArray) { + + constructor(hue: Float, saturation: Float, value: Float) : this(floatArrayOf(hue, saturation, value)) + constructor(hue: Double, saturation: Double, value: Double) : this(floatArrayOf(hue.toFloat(), saturation.toFloat(), value.toFloat())) + + /** Accessor for the hue component of the [HSVColorContainer]. */ + val hue: Float + get() = this.colors[0] + + /** Accessor for the saturation component of the [HSVColorContainer]. */ + val saturation: Float + get() = this.colors[1] + + /** Accessor for the value component of the [HSVColorContainer]. */ + val value: Float + get() = this.colors[2] +} \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/MutableRGBFloatColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/rgb/MutableRGBFloatColorContainer.kt similarity index 98% rename from vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/MutableRGBFloatColorContainer.kt rename to vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/rgb/MutableRGBFloatColorContainer.kt index 26a2f3721..aa5eb78a2 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/MutableRGBFloatColorContainer.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/rgb/MutableRGBFloatColorContainer.kt @@ -1,4 +1,4 @@ -package org.vitrivr.engine.core.model.color +package org.vitrivr.engine.core.model.color.rgb import kotlin.math.max import kotlin.math.min diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBByteColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/rgb/RGBByteColorContainer.kt similarity index 86% rename from vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBByteColorContainer.kt rename to vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/rgb/RGBByteColorContainer.kt index 8db668c17..dfe77f15e 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBByteColorContainer.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/rgb/RGBByteColorContainer.kt @@ -1,4 +1,4 @@ -package org.vitrivr.engine.core.model.color +package org.vitrivr.engine.core.model.color.rgb /** * A container for RGB colors. @@ -31,7 +31,9 @@ value class RGBByteColorContainer(private val rgb: Int) { get() = (this.rgb and 0xFF).toUByte() /** + * Converts this [RGBByteColorContainer] to an [RGBFloatColorContainer]. * + * @return [RGBFloatColorContainer] representation of this [RGBByteColorContainer]. */ fun toFloatContainer(): RGBFloatColorContainer = RGBFloatColorContainer(this.red.toFloat() / 255f, this.green.toFloat() / 255f, this.blue.toFloat() / 255f) diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBFloatColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/rgb/RGBFloatColorContainer.kt similarity index 98% rename from vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBFloatColorContainer.kt rename to vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/rgb/RGBFloatColorContainer.kt index 3218bc69e..a478f6086 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBFloatColorContainer.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/rgb/RGBFloatColorContainer.kt @@ -1,4 +1,4 @@ -package org.vitrivr.engine.core.model.color +package org.vitrivr.engine.core.model.color.rgb import org.vitrivr.engine.core.model.types.Value import kotlin.math.max diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/AverageImageContentAggregator.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/AverageImageContentAggregator.kt index 397cb4855..c857e3bda 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/AverageImageContentAggregator.kt +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/AverageImageContentAggregator.kt @@ -2,8 +2,8 @@ package org.vitrivr.engine.index.aggregators.image import org.vitrivr.engine.core.context.Context import org.vitrivr.engine.core.context.IndexContext -import org.vitrivr.engine.core.model.color.MutableRGBFloatColorContainer -import org.vitrivr.engine.core.model.color.RGBByteColorContainer +import org.vitrivr.engine.core.model.color.rgb.MutableRGBFloatColorContainer +import org.vitrivr.engine.core.model.color.rgb.RGBByteColorContainer import org.vitrivr.engine.core.model.content.decorators.SourcedContent import org.vitrivr.engine.core.model.content.decorators.TemporalContent import org.vitrivr.engine.core.model.content.element.ContentElement diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/RepresentativeImageContentAggregator.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/RepresentativeImageContentAggregator.kt index 51ddbec61..e38d944da 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/RepresentativeImageContentAggregator.kt +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/RepresentativeImageContentAggregator.kt @@ -2,8 +2,8 @@ package org.vitrivr.engine.index.aggregators.image import org.vitrivr.engine.core.context.Context import org.vitrivr.engine.core.context.IndexContext -import org.vitrivr.engine.core.model.color.MutableRGBFloatColorContainer -import org.vitrivr.engine.core.model.color.RGBByteColorContainer +import org.vitrivr.engine.core.model.color.rgb.MutableRGBFloatColorContainer +import org.vitrivr.engine.core.model.color.rgb.RGBByteColorContainer import org.vitrivr.engine.core.model.content.element.ContentElement import org.vitrivr.engine.core.model.content.element.ImageContent import org.vitrivr.engine.core.model.retrievable.Ingested diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogram.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogram.kt new file mode 100644 index 000000000..609cfe14b --- /dev/null +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogram.kt @@ -0,0 +1,151 @@ +package org.vitrivr.engine.module.features.feature.huehistogram + +import org.vitrivr.engine.core.context.IndexContext +import org.vitrivr.engine.core.context.QueryContext +import org.vitrivr.engine.core.model.color.ColorConverter +import org.vitrivr.engine.core.model.color.hsv.HSVColorContainer +import org.vitrivr.engine.core.model.color.rgb.RGBByteColorContainer +import org.vitrivr.engine.core.model.content.Content +import org.vitrivr.engine.core.model.content.element.ContentElement +import org.vitrivr.engine.core.model.content.element.ImageContent +import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor +import org.vitrivr.engine.core.model.metamodel.Analyser +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.query.Query +import org.vitrivr.engine.core.model.query.proximity.ProximityQuery +import org.vitrivr.engine.core.model.retrievable.Retrievable +import org.vitrivr.engine.core.model.types.Value +import org.vitrivr.engine.core.operators.Operator +import org.vitrivr.engine.core.operators.ingest.Extractor +import org.vitrivr.engine.core.operators.retrieve.Retriever +import java.util.* +import kotlin.reflect.KClass + +/** + * A hue histogram [Analyser] for [ImageContent] objects. + * + * Migrated from Cineast. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class HueHistogram : Analyser { + + companion object { + private const val VECTOR_SIZE = 16 + } + + override val contentClasses: Set>> = setOf(ImageContent::class) + override val descriptorClass: KClass = FloatVectorDescriptor::class + + /** + * Generates a prototypical [FloatVectorDescriptor] for this [HueHistogram]. + * + * @param field [Schema.Field] to create the prototype for. + * @return [FloatVectorDescriptor] + */ + override fun prototype(field: Schema.Field<*, *>): FloatVectorDescriptor = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.FloatVector(VECTOR_SIZE)) + + /** + * Generates and returns a new [HueHistogramExtractor] instance for this [HueHistogram]. + * + * @param name The name of the [HueHistogramExtractor]. + * @param input The [Operator] that acts as input to the new [Extractor]. + * @param context The [IndexContext] to use with the [Extractor]. + * + * @return A new [Extractor] instance for this [Analyser] + * @throws [UnsupportedOperationException], if this [Analyser] does not support the creation of an [Extractor] instance. + */ + override fun newExtractor(name: String, input: Operator, context: IndexContext) = HueHistogramExtractor(input, this, null) + + /** + * Generates and returns a new [HueHistogramExtractor] instance for this [HueHistogram]. + * + * @param field The [Schema.Field] to create an [Extractor] for. + * @param input The [Operator] that acts as input to the new [Extractor]. + * @param context The [IndexContext] to use with the [Extractor]. + * + * @return A new [Extractor] instance for this [Analyser] + * @throws [UnsupportedOperationException], if this [Analyser] does not support the creation of an [Extractor] instance. + */ + override fun newExtractor(field: Schema.Field, input: Operator, context: IndexContext) = HueHistogramExtractor(input, this, field) + + /** + * Generates and returns a new [HueHistogramRetriever] instance for this [HueHistogram]. + * + * @param field The [Schema.Field] to create an [Retriever] for. + * @param query The [Query] to use with the [Retriever]. + * @param context The [QueryContext] to use with the [Retriever]. + * + * @return A new [HueHistogramRetriever] instance for this [HueHistogram] + */ + override fun newRetrieverForQuery(field: Schema.Field, query: Query, context: QueryContext): HueHistogramRetriever { + require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } + require(query is ProximityQuery<*> && query.value is Value.FloatVector) { "The query is not a ProximityQuery." } + @Suppress("UNCHECKED_CAST") + return HueHistogramRetriever(field, query as ProximityQuery, context) + } + + /** + * Generates and returns a new [HueHistogramRetriever] instance for this [HueHistogram]. + * + * Invoking this method involves converting the provided [FloatVectorDescriptor] into a [ProximityQuery] that can be used to retrieve similar [ImageContent] elements. + * + * @param field The [Schema.Field] to create an [Retriever] for. + * @param descriptors An array of [FloatVectorDescriptor] elements to use with the [Retriever] + * @param context The [QueryContext] to use with the [Retriever] + */ + override fun newRetrieverForDescriptors(field: Schema.Field, descriptors: Collection, context: QueryContext): HueHistogramRetriever { + require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } + + /* Prepare query parameters. */ + val k = context.getProperty(field.fieldName, "limit")?.toLongOrNull() ?: 1000L + val fetchVector = context.getProperty(field.fieldName, "returnDescriptor")?.toBooleanStrictOrNull() ?: false + + /* Return retriever. */ + return this.newRetrieverForQuery(field, ProximityQuery(value = descriptors.first().vector, k = k, fetchVector = fetchVector), context) + } + + /** + * Generates and returns a new [HueHistogramRetriever] instance for this [HueHistogram]. + * + * Invoking this method involves converting the provided [ImageContent] and the [QueryContext] into a [FloatVectorDescriptor] + * that can be used to retrieve similar [ImageContent] elements. + * + * @param field The [Schema.Field] to create an [Retriever] for. + * @param content An array of [Content] elements to use with the [Retriever] + * @param context The [QueryContext] to use with the [Retriever] + */ + override fun newRetrieverForContent(field: Schema.Field, content: Collection, context: QueryContext) = this.newRetrieverForDescriptors(field, content.map { this.analyse(it) }, context) + + /** + * Performs the [HueHistogram] analysis on the provided [ImageContent] and returns a [FloatVectorDescriptor] that represents the result. + * + * @param content [ImageContent] to be analysed. + * @return [FloatVectorDescriptor] result of the analysis. + */ + fun analyse(content: ImageContent): FloatVectorDescriptor { + /* Generate histogram. */ + val hist = FloatArray(VECTOR_SIZE) + val colors = content.content.getRGB(0, 0, content.content.width, content.content.height, null, 0, content.content.width) + for (color in colors) { + val container: HSVColorContainer = ColorConverter.rgbToHsv(RGBByteColorContainer(color)) + if (container.saturation > 0.2f && container.value > 0.3f) { + val h: Float = container.hue * hist.size + val idx = h.toInt() + hist[idx] += h - idx + hist[(idx + 1) % hist.size] += idx + 1 - h + } + } + + /* Normalize hist. */ + val sum = hist.sum() + if (sum > 1f) { + for (i in hist.indices) { + hist[i] /= sum + } + } + + return FloatVectorDescriptor(vector = Value.FloatVector(hist)) + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogramExtractor.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogramExtractor.kt new file mode 100644 index 000000000..f16974cb5 --- /dev/null +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogramExtractor.kt @@ -0,0 +1,45 @@ +package org.vitrivr.engine.module.features.feature.huehistogram + +import org.vitrivr.engine.core.features.AbstractExtractor +import org.vitrivr.engine.core.features.metadata.source.file.FileSourceMetadataExtractor +import org.vitrivr.engine.core.model.content.ContentType +import org.vitrivr.engine.core.model.content.element.ImageContent +import org.vitrivr.engine.core.model.descriptor.Descriptor +import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.retrievable.Retrievable +import org.vitrivr.engine.core.operators.Operator +import org.vitrivr.engine.core.operators.ingest.Extractor +import org.vitrivr.engine.core.source.file.FileSource +import org.vitrivr.engine.module.features.feature.ehd.EHD + +/** + * An [Extractor] implementation for the [HueHistogram] analyser. + * + * @see [EHD] + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class HueHistogramExtractor(input: Operator, analyser: HueHistogram, field: Schema.Field?) : AbstractExtractor(input, analyser, field) { + /** + * Internal method to check, if [Retrievable] matches this [Extractor] and should thus be processed. + * + * [FileSourceMetadataExtractor] implementation only works with [Retrievable] that contain a [FileSource]. + * + * @param retrievable The [Retrievable] to check. + * @return True on match, false otherwise, + */ + override fun matches(retrievable: Retrievable): Boolean = retrievable.content.any { it.type == ContentType.BITMAP_IMAGE } + + /** + * Internal method to perform extraction on [Retrievable]. + * + * @param retrievable The [Retrievable] to process. + * @return List of resulting [Descriptor]s. + */ + override fun extract(retrievable: Retrievable): List { + val content = retrievable.content.filterIsInstance() + return content.map { (this.analyser as HueHistogram).analyse(it).copy(retrievableId = retrievable.id, field = this.field) } + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogramRetriever.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogramRetriever.kt new file mode 100644 index 000000000..0edbd3264 --- /dev/null +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogramRetriever.kt @@ -0,0 +1,42 @@ +package org.vitrivr.engine.module.features.feature.huehistogram + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.flow.flow +import org.vitrivr.engine.core.context.QueryContext +import org.vitrivr.engine.core.features.AbstractRetriever +import org.vitrivr.engine.core.math.correspondence.LinearCorrespondence +import org.vitrivr.engine.core.model.content.element.ImageContent +import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.query.proximity.ProximityQuery +import org.vitrivr.engine.core.model.retrievable.attributes.DistanceAttribute +import org.vitrivr.engine.core.model.types.Value +import org.vitrivr.engine.core.operators.retrieve.Retriever + +/** + * [Retriever] implementation for the [HueHistogram] analyser. + * + * @see [HueHistogram] + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class HueHistogramRetriever(field: Schema.Field, query: ProximityQuery, context: QueryContext) : AbstractRetriever(field, query, context) { + + companion object { + /** [LinearCorrespondence] for [HueHistogramRetriever]. Maximum distance is taken from Cineast implementation. */ + private val CORRESPONDENCE = LinearCorrespondence(16f) + } + + override fun toFlow(scope: CoroutineScope) = flow { + this@HueHistogramRetriever.reader.queryAndJoin(this@HueHistogramRetriever.query).forEach { + val distance = it.filteredAttribute() + if (distance != null) { + it.addAttribute(CORRESPONDENCE(distance)) + } else { + this@HueHistogramRetriever.logger.warn { "No distance attribute found for descriptor ${it.id}." } + } + emit(it) + } + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser b/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser index 446faf6f8..032bde4d2 100644 --- a/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser +++ b/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser @@ -1,3 +1,4 @@ org.vitrivr.engine.module.features.feature.ehd.EHD +org.vitrivr.engine.module.features.feature.huehistogram.HueHistogram org.vitrivr.engine.module.features.feature.external.implementations.dino.DINO org.vitrivr.engine.module.features.feature.external.implementations.clip.CLIP \ No newline at end of file diff --git a/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/api/ImageCaptioningApi.kt b/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/api/ImageCaptioningApi.kt index a2602c5e6..a3a073e73 100644 --- a/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/api/ImageCaptioningApi.kt +++ b/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/api/ImageCaptioningApi.kt @@ -82,7 +82,7 @@ class ImageCaptioningApi(host: String, model: String, timeoutMs: Long, pollingIn */ override suspend fun pollBatchedJob(jobId: String): JobResult> = try { this.imageCaptioningApi.getBatchedJobResultsApiTasksImageCaptioningBatchedJobsJobGet(jobId).body().let { result -> - val values = result.result?.map { Value.Text(it.caption.trim() ?: "") } + val values = result.result?.map { Value.Text(it.caption.trim()) } JobResult(result.status, values) } } catch (e: Throwable) { From cc5d1260d54937c8615f599bc0209a9f334e6355 Mon Sep 17 00:00:00 2001 From: Ralph Gasser Date: Wed, 21 Aug 2024 10:06:24 +0200 Subject: [PATCH 06/20] DenseRetriever now has a configurable correspondence function. Various feature modules now return a DenseRetriever instance instead of their own implementation. Signed-off-by: Ralph Gasser --- .../features/averagecolor/AverageColor.kt | 19 ++++----- .../averagecolor/AverageColorRetriever.kt | 42 ------------------- .../core/features/dense/DenseRetriever.kt | 15 +++++-- .../correspondence/BoundedCorrespondence.kt | 14 +++++++ .../correspondence/LinearCorrespondence.kt | 4 +- .../engine/core/util/math/ScoringFunctions.kt | 37 ---------------- .../engine/module/features/feature/ehd/EHD.kt | 19 +++++---- .../features/feature/ehd/EHDRetriever.kt | 42 ------------------- .../external/implementations/clip/CLIP.kt | 4 +- .../external/implementations/dino/DINO.kt | 4 +- .../feature/huehistogram/HueHistogram.kt | 16 +++---- .../huehistogram/HueHistogramRetriever.kt | 42 ------------------- .../implementations/dense/DenseEmbedding.kt | 3 +- 13 files changed, 62 insertions(+), 199 deletions(-) delete mode 100644 vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColorRetriever.kt create mode 100644 vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/math/correspondence/BoundedCorrespondence.kt delete mode 100644 vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/util/math/ScoringFunctions.kt delete mode 100644 vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/ehd/EHDRetriever.kt delete mode 100644 vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogramRetriever.kt diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColor.kt index 4b0f52c8d..cedafc42f 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColor.kt @@ -2,6 +2,8 @@ package org.vitrivr.engine.core.features.averagecolor import org.vitrivr.engine.core.context.IndexContext import org.vitrivr.engine.core.context.QueryContext +import org.vitrivr.engine.core.features.dense.DenseRetriever +import org.vitrivr.engine.core.math.correspondence.LinearCorrespondence import org.vitrivr.engine.core.model.color.rgb.MutableRGBFloatColorContainer import org.vitrivr.engine.core.model.color.rgb.RGBByteColorContainer import org.vitrivr.engine.core.model.color.rgb.RGBFloatColorContainer @@ -66,7 +68,7 @@ class AverageColor : Analyser { override fun newExtractor(name: String, input: Operator, context: IndexContext): Extractor = AverageColorExtractor(input, this, null) /** - * Generates and returns a new [AverageColorRetriever] instance for this [AverageColor]. + * Generates and returns a new [DenseRetriever] instance for this [AverageColor]. * * @param field The [Schema.Field] to create an [Retriever] for. * @param query The [Query] to use with the [Retriever]. @@ -74,15 +76,14 @@ class AverageColor : Analyser { * * @return A new [Retriever] instance for this [Analyser] */ - override fun newRetrieverForQuery(field: Schema.Field, query: Query, context: QueryContext): AverageColorRetriever { - require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } + override fun newRetrieverForQuery(field: Schema.Field, query: Query, context: QueryContext): DenseRetriever { require(query is ProximityQuery<*> && query.value is Value.FloatVector) { "The query is not a ProximityQuery." } @Suppress("UNCHECKED_CAST") - return AverageColorRetriever(field, query as ProximityQuery, context) + return DenseRetriever(field, query as ProximityQuery, context, LinearCorrespondence(3f)) } /** - * Generates and returns a new [AverageColorRetriever] instance for this [AverageColor]. + * Generates and returns a new [DenseRetriever] instance for this [AverageColor]. * * Invoking this method involves converting the provided [FloatVectorDescriptor] into a [ProximityQuery] that can be used to retrieve similar [ImageContent] elements. * @@ -90,9 +91,7 @@ class AverageColor : Analyser { * @param descriptors An array of [FloatVectorDescriptor] elements to use with the [Retriever] * @param context The [QueryContext] to use with the [Retriever] */ - override fun newRetrieverForDescriptors(field: Schema.Field, descriptors: Collection, context: QueryContext): AverageColorRetriever { - require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } - + override fun newRetrieverForDescriptors(field: Schema.Field, descriptors: Collection, context: QueryContext): DenseRetriever { /* Prepare query parameters. */ val k = context.getProperty(field.fieldName, "limit")?.toLongOrNull() ?: 1000L val fetchVector = context.getProperty(field.fieldName, "returnDescriptor")?.toBooleanStrictOrNull() ?: false @@ -102,7 +101,7 @@ class AverageColor : Analyser { } /** - * Generates and returns a new [AverageColorRetriever] instance for this [AverageColor]. + * Generates and returns a new [DenseRetriever] instance for this [AverageColor]. * * Invoking this method involves converting the provided [ImageContent] and the [QueryContext] into a [FloatVectorDescriptor] * that can be used to retrieve similar [ImageContent] elements. @@ -111,7 +110,7 @@ class AverageColor : Analyser { * @param content An array of [Content] elements to use with the [Retriever] * @param context The [QueryContext] to use with the [Retriever] */ - override fun newRetrieverForContent(field: Schema.Field, content: Collection, context: QueryContext): AverageColorRetriever = + override fun newRetrieverForContent(field: Schema.Field, content: Collection, context: QueryContext): DenseRetriever = this.newRetrieverForDescriptors(field, this.analyse(content), context) /** diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColorRetriever.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColorRetriever.kt deleted file mode 100644 index 126640442..000000000 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColorRetriever.kt +++ /dev/null @@ -1,42 +0,0 @@ -package org.vitrivr.engine.core.features.averagecolor - -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.flow.flow -import org.vitrivr.engine.core.context.QueryContext -import org.vitrivr.engine.core.features.AbstractRetriever -import org.vitrivr.engine.core.math.correspondence.LinearCorrespondence -import org.vitrivr.engine.core.model.content.element.ImageContent -import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor -import org.vitrivr.engine.core.model.metamodel.Schema -import org.vitrivr.engine.core.model.query.proximity.ProximityQuery -import org.vitrivr.engine.core.model.retrievable.attributes.DistanceAttribute -import org.vitrivr.engine.core.model.types.Value -import org.vitrivr.engine.core.operators.retrieve.Retriever - -/** - * [Retriever] implementation for the [AverageColor] analyser. - * - * @see [AverageColor] - * - * @author Luca Rossetto - * @version 1.2.0 - */ -class AverageColorRetriever(field: Schema.Field, query: ProximityQuery, context: QueryContext) : AbstractRetriever(field, query, context) { - - companion object { - /** [LinearCorrespondence] for [AverageColorRetriever]. */ - private val CORRESPONDENCE = LinearCorrespondence(3f) - } - - override fun toFlow(scope: CoroutineScope) = flow { - this@AverageColorRetriever.reader.queryAndJoin(this@AverageColorRetriever.query).forEach { - val distance = it.filteredAttribute() - if (distance != null) { - it.addAttribute(CORRESPONDENCE(distance)) - } else { - this@AverageColorRetriever.logger.warn { "No distance attribute found for descriptor ${it.id}." } - } - emit(it) - } - } -} diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/dense/DenseRetriever.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/dense/DenseRetriever.kt index 94fe546cf..ea97b2da9 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/dense/DenseRetriever.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/dense/DenseRetriever.kt @@ -4,11 +4,13 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.flow.flow import org.vitrivr.engine.core.context.QueryContext import org.vitrivr.engine.core.features.AbstractRetriever +import org.vitrivr.engine.core.math.correspondence.CorrespondenceFunction import org.vitrivr.engine.core.model.content.element.ContentElement import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.query.proximity.ProximityQuery -import org.vitrivr.engine.core.util.math.ScoringFunctions +import org.vitrivr.engine.core.model.retrievable.attributes.DistanceAttribute +import org.vitrivr.engine.core.model.retrievable.attributes.ScoreAttribute /** * [DenseRetriever] implementation for proximity-based retrieval on float vector embeddings. @@ -24,10 +26,17 @@ import org.vitrivr.engine.core.util.math.ScoringFunctions * @author Fynn Faber * @version 1.0.0 */ -class DenseRetriever>(field: Schema.Field, query: ProximityQuery<*>, context: QueryContext) : AbstractRetriever(field, query, context) { +class DenseRetriever>(field: Schema.Field, query: ProximityQuery<*>, context: QueryContext, val correspondence: CorrespondenceFunction) : + AbstractRetriever(field, query, context) { override fun toFlow(scope: CoroutineScope) = flow { this@DenseRetriever.reader.queryAndJoin(this@DenseRetriever.query).forEach { - it.addAttribute(ScoringFunctions.bounded(it, 0.0f, 2.0f)) + val distance = it.filteredAttribute() + if (distance != null) { + it.addAttribute(this@DenseRetriever.correspondence(distance)) + } else { + this@DenseRetriever.logger.warn { "No distance attribute found for descriptor ${it.id}." } + it.addAttribute(ScoreAttribute.Similarity(0.0f)) + } emit(it) } } diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/math/correspondence/BoundedCorrespondence.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/math/correspondence/BoundedCorrespondence.kt new file mode 100644 index 000000000..a55644767 --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/math/correspondence/BoundedCorrespondence.kt @@ -0,0 +1,14 @@ +package org.vitrivr.engine.core.math.correspondence + +import org.vitrivr.engine.core.model.retrievable.attributes.DistanceAttribute +import org.vitrivr.engine.core.model.retrievable.attributes.ScoreAttribute + +/** + * A [CorrespondenceFunction] that is based on and upper and lower bound for the distance value. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class BoundedCorrespondence(private val min: Float = 0.0f, private val max: Float = 1.0f) : CorrespondenceFunction { + override fun invoke(distance: DistanceAttribute): ScoreAttribute.Similarity = ScoreAttribute.Similarity((this.max - distance.distance) / (this.max - this.min)) +} \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/math/correspondence/LinearCorrespondence.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/math/correspondence/LinearCorrespondence.kt index 7a0b2a7da..c95717a99 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/math/correspondence/LinearCorrespondence.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/math/correspondence/LinearCorrespondence.kt @@ -9,6 +9,6 @@ import org.vitrivr.engine.core.model.retrievable.attributes.ScoreAttribute * @author Ralph Gasser * @version 1.0.0 */ -class LinearCorrespondence(private val maximumDistance: Float) : CorrespondenceFunction { - override fun invoke(distance: DistanceAttribute): ScoreAttribute.Similarity = ScoreAttribute.Similarity(1.0f - (distance.distance / this.maximumDistance)) +class LinearCorrespondence(private val max: Float) : CorrespondenceFunction { + override fun invoke(distance: DistanceAttribute): ScoreAttribute.Similarity = ScoreAttribute.Similarity(1.0f - (distance.distance / this.max)) } \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/util/math/ScoringFunctions.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/util/math/ScoringFunctions.kt deleted file mode 100644 index faf5a316e..000000000 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/util/math/ScoringFunctions.kt +++ /dev/null @@ -1,37 +0,0 @@ -package org.vitrivr.engine.core.util.math - -import org.vitrivr.engine.core.model.retrievable.Retrieved -import org.vitrivr.engine.core.model.retrievable.attributes.DistanceAttribute -import org.vitrivr.engine.core.model.retrievable.attributes.ScoreAttribute - -/** - * A collection of [ScoringFunctions] that can be used to score [Retrieved] object that come with a [DistanceAttribute]. - * - * @author Ralph Gasser - * @version 1.0.0 - */ -object ScoringFunctions { - - /** - * A scoring function that assumes a defined minimum and maximum and normalizes the distance between these two values. - * - * @param retrieved [Retrieved] object to score. - * @param min Minimum value. Default is 0.0. - * @param max Maximum value. Default is 1.0. - */ - fun bounded(retrieved: Retrieved, min: Float = 0.0f, max: Float = 1.0f): ScoreAttribute { - val distance = retrieved.filteredAttribute()?.distance ?: return ScoreAttribute.Similarity(0.0f) - return ScoreAttribute.Similarity((max-distance) / (max - min)) - } - - /** - * A scoring functions that assumes a defined maximum and subtracts the distance from that maximum to obtain a score. - * - * @param retrieved [Retrieved] object to score. - * @param max Maximum value. Default is 1.0. - */ - fun max(retrieved: Retrieved, max: Float = 1.0f): ScoreAttribute { - val distance = retrieved.filteredAttribute()?.distance ?: return ScoreAttribute.Unbound(0.0f) - return ScoreAttribute.Unbound(max - distance) - } -} diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/ehd/EHD.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/ehd/EHD.kt index 70c0a6178..1c752cd9c 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/ehd/EHD.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/ehd/EHD.kt @@ -4,6 +4,8 @@ import boofcv.io.image.ConvertBufferedImage import boofcv.struct.image.GrayU8 import org.vitrivr.engine.core.context.IndexContext import org.vitrivr.engine.core.context.QueryContext +import org.vitrivr.engine.core.features.dense.DenseRetriever +import org.vitrivr.engine.core.math.correspondence.LinearCorrespondence import org.vitrivr.engine.core.model.content.Content import org.vitrivr.engine.core.model.content.element.ContentElement import org.vitrivr.engine.core.model.content.element.ImageContent @@ -102,23 +104,23 @@ class EHD : Analyser { override fun newExtractor(field: Schema.Field, input: Operator, context: IndexContext) = EHDExtractor(input, this, field) /** - * Generates and returns a new [EHDRetriever] instance for this [EHD]. + * Generates and returns a new [DenseRetriever] instance for this [EHD]. * * @param field The [Schema.Field] to create an [Retriever] for. * @param query The [Query] to use with the [Retriever]. * @param context The [QueryContext] to use with the [Retriever]. * - * @return A new [EHDRetriever] instance for this [EHD] + * @return A new [DenseRetriever] instance for this [EHD] */ - override fun newRetrieverForQuery(field: Schema.Field, query: Query, context: QueryContext): EHDRetriever { + override fun newRetrieverForQuery(field: Schema.Field, query: Query, context: QueryContext): DenseRetriever { require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } require(query is ProximityQuery<*> && query.value is Value.FloatVector) { "The query is not a ProximityQuery." } @Suppress("UNCHECKED_CAST") - return EHDRetriever(field, query as ProximityQuery, context) + return DenseRetriever(field, query as ProximityQuery, context, LinearCorrespondence(4f)) } /** - * Generates and returns a new [EHDRetriever] instance for this [EHD]. + * Generates and returns a new [DenseRetriever] instance for this [EHD]. * * Invoking this method involves converting the provided [FloatVectorDescriptor] into a [ProximityQuery] that can be used to retrieve similar [ImageContent] elements. * @@ -126,7 +128,7 @@ class EHD : Analyser { * @param descriptors An array of [FloatVectorDescriptor] elements to use with the [Retriever] * @param context The [QueryContext] to use with the [Retriever] */ - override fun newRetrieverForDescriptors(field: Schema.Field, descriptors: Collection, context: QueryContext): EHDRetriever { + override fun newRetrieverForDescriptors(field: Schema.Field, descriptors: Collection, context: QueryContext): DenseRetriever { require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } /* Prepare query parameters. */ @@ -138,7 +140,7 @@ class EHD : Analyser { } /** - * Generates and returns a new [EHDRetriever] instance for this [EHD]. + * Generates and returns a new [DenseRetriever] instance for this [EHD]. * * Invoking this method involves converting the provided [ImageContent] and the [QueryContext] into a [FloatVectorDescriptor] * that can be used to retrieve similar [ImageContent] elements. @@ -147,8 +149,7 @@ class EHD : Analyser { * @param content An array of [Content] elements to use with the [Retriever] * @param context The [QueryContext] to use with the [Retriever] */ - override fun newRetrieverForContent(field: Schema.Field, content: Collection, context: QueryContext): EHDRetriever = - this.newRetrieverForDescriptors(field, content.map { this.analyse(it) }, context) + override fun newRetrieverForContent(field: Schema.Field, content: Collection, context: QueryContext) = this.newRetrieverForDescriptors(field, content.map { this.analyse(it) }, context) /** * Performs the [EHD] analysis on the provided [ImageContent] and returns a [FloatVectorDescriptor] that represents the result. diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/ehd/EHDRetriever.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/ehd/EHDRetriever.kt deleted file mode 100644 index 30ee064ce..000000000 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/ehd/EHDRetriever.kt +++ /dev/null @@ -1,42 +0,0 @@ -package org.vitrivr.engine.module.features.feature.ehd - -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.flow.flow -import org.vitrivr.engine.core.context.QueryContext -import org.vitrivr.engine.core.features.AbstractRetriever -import org.vitrivr.engine.core.math.correspondence.LinearCorrespondence -import org.vitrivr.engine.core.model.content.element.ImageContent -import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor -import org.vitrivr.engine.core.model.metamodel.Schema -import org.vitrivr.engine.core.model.query.proximity.ProximityQuery -import org.vitrivr.engine.core.model.retrievable.attributes.DistanceAttribute -import org.vitrivr.engine.core.model.types.Value -import org.vitrivr.engine.core.operators.retrieve.Retriever - -/** - * [Retriever] implementation for the [EHD] analyser. - * - * @see [EHD] - * - * @author Ralph Gasser - * @version 1.0.0 - */ -class EHDRetriever(field: Schema.Field, query: ProximityQuery, context: QueryContext) : AbstractRetriever(field, query, context) { - - companion object { - /** [LinearCorrespondence] for [EHDRetriever]. Maximum distance is taken from Cineast implementation. */ - private val CORRESPONDENCE = LinearCorrespondence(16 / 4f) - } - - override fun toFlow(scope: CoroutineScope) = flow { - this@EHDRetriever.reader.queryAndJoin(this@EHDRetriever.query).forEach { - val distance = it.filteredAttribute() - if (distance != null) { - it.addAttribute(CORRESPONDENCE(distance)) - } else { - this@EHDRetriever.logger.warn { "No distance attribute found for descriptor ${it.id}." } - } - emit(it) - } - } -} \ No newline at end of file diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/external/implementations/clip/CLIP.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/external/implementations/clip/CLIP.kt index 6ef5eab20..ab4f3dcd4 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/external/implementations/clip/CLIP.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/external/implementations/clip/CLIP.kt @@ -3,6 +3,7 @@ package org.vitrivr.engine.module.features.feature.external.implementations.clip import org.vitrivr.engine.core.context.IndexContext import org.vitrivr.engine.core.context.QueryContext import org.vitrivr.engine.core.features.dense.DenseRetriever +import org.vitrivr.engine.core.math.correspondence.BoundedCorrespondence import org.vitrivr.engine.core.model.content.element.ContentElement import org.vitrivr.engine.core.model.content.element.ImageContent import org.vitrivr.engine.core.model.content.element.TextContent @@ -106,10 +107,9 @@ class CLIP : ExternalAnalyser, FloatVectorDescriptor>() { * @throws [UnsupportedOperationException], if this [CLIP] does not support the creation of an [Retriever] instance. */ override fun newRetrieverForQuery(field: Schema.Field, FloatVectorDescriptor>, query: Query, context: QueryContext): DenseRetriever> { - require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } require(query is ProximityQuery<*> && query.value is Value.FloatVector) { "The query is not a ProximityQuery." } @Suppress("UNCHECKED_CAST") - return DenseRetriever(field, query as ProximityQuery, context) + return DenseRetriever(field, query as ProximityQuery, context, BoundedCorrespondence(0.0f, 2.0f)) } /** diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/external/implementations/dino/DINO.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/external/implementations/dino/DINO.kt index 34fe388a8..e23f1f19b 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/external/implementations/dino/DINO.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/external/implementations/dino/DINO.kt @@ -3,6 +3,7 @@ package org.vitrivr.engine.module.features.feature.external.implementations.dino import org.vitrivr.engine.core.context.IndexContext import org.vitrivr.engine.core.context.QueryContext import org.vitrivr.engine.core.features.dense.DenseRetriever +import org.vitrivr.engine.core.math.correspondence.BoundedCorrespondence import org.vitrivr.engine.core.model.content.Content import org.vitrivr.engine.core.model.content.element.ContentElement import org.vitrivr.engine.core.model.content.element.ImageContent @@ -93,10 +94,9 @@ class DINO : ExternalAnalyser() { * @throws [UnsupportedOperationException], if this [Analyser] does not support the creation of an [Retriever] instance. */ override fun newRetrieverForQuery(field: Schema.Field, query: Query, context: QueryContext): DenseRetriever { - require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } require(query is ProximityQuery<*> && query.value is Value.FloatVector) { "The query is not a ProximityQuery." } @Suppress("UNCHECKED_CAST") - return DenseRetriever(field, query as ProximityQuery, context) + return DenseRetriever(field, query as ProximityQuery, context, BoundedCorrespondence(0.0f, 2.0f)) } /** diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogram.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogram.kt index 609cfe14b..6aed6dc87 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogram.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogram.kt @@ -2,6 +2,8 @@ package org.vitrivr.engine.module.features.feature.huehistogram import org.vitrivr.engine.core.context.IndexContext import org.vitrivr.engine.core.context.QueryContext +import org.vitrivr.engine.core.features.dense.DenseRetriever +import org.vitrivr.engine.core.math.correspondence.LinearCorrespondence import org.vitrivr.engine.core.model.color.ColorConverter import org.vitrivr.engine.core.model.color.hsv.HSVColorContainer import org.vitrivr.engine.core.model.color.rgb.RGBByteColorContainer @@ -71,23 +73,23 @@ class HueHistogram : Analyser { override fun newExtractor(field: Schema.Field, input: Operator, context: IndexContext) = HueHistogramExtractor(input, this, field) /** - * Generates and returns a new [HueHistogramRetriever] instance for this [HueHistogram]. + * Generates and returns a new [DenseRetriever] instance for this [HueHistogram]. * * @param field The [Schema.Field] to create an [Retriever] for. * @param query The [Query] to use with the [Retriever]. * @param context The [QueryContext] to use with the [Retriever]. * - * @return A new [HueHistogramRetriever] instance for this [HueHistogram] + * @return A new [DenseRetriever] instance for this [HueHistogram] */ - override fun newRetrieverForQuery(field: Schema.Field, query: Query, context: QueryContext): HueHistogramRetriever { + override fun newRetrieverForQuery(field: Schema.Field, query: Query, context: QueryContext): DenseRetriever { require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } require(query is ProximityQuery<*> && query.value is Value.FloatVector) { "The query is not a ProximityQuery." } @Suppress("UNCHECKED_CAST") - return HueHistogramRetriever(field, query as ProximityQuery, context) + return DenseRetriever(field, query as ProximityQuery, context, LinearCorrespondence(16f)) } /** - * Generates and returns a new [HueHistogramRetriever] instance for this [HueHistogram]. + * Generates and returns a new [DenseRetriever] instance for this [HueHistogram]. * * Invoking this method involves converting the provided [FloatVectorDescriptor] into a [ProximityQuery] that can be used to retrieve similar [ImageContent] elements. * @@ -95,7 +97,7 @@ class HueHistogram : Analyser { * @param descriptors An array of [FloatVectorDescriptor] elements to use with the [Retriever] * @param context The [QueryContext] to use with the [Retriever] */ - override fun newRetrieverForDescriptors(field: Schema.Field, descriptors: Collection, context: QueryContext): HueHistogramRetriever { + override fun newRetrieverForDescriptors(field: Schema.Field, descriptors: Collection, context: QueryContext): DenseRetriever { require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } /* Prepare query parameters. */ @@ -107,7 +109,7 @@ class HueHistogram : Analyser { } /** - * Generates and returns a new [HueHistogramRetriever] instance for this [HueHistogram]. + * Generates and returns a new [DenseRetriever] instance for this [HueHistogram]. * * Invoking this method involves converting the provided [ImageContent] and the [QueryContext] into a [FloatVectorDescriptor] * that can be used to retrieve similar [ImageContent] elements. diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogramRetriever.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogramRetriever.kt deleted file mode 100644 index 0edbd3264..000000000 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogramRetriever.kt +++ /dev/null @@ -1,42 +0,0 @@ -package org.vitrivr.engine.module.features.feature.huehistogram - -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.flow.flow -import org.vitrivr.engine.core.context.QueryContext -import org.vitrivr.engine.core.features.AbstractRetriever -import org.vitrivr.engine.core.math.correspondence.LinearCorrespondence -import org.vitrivr.engine.core.model.content.element.ImageContent -import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor -import org.vitrivr.engine.core.model.metamodel.Schema -import org.vitrivr.engine.core.model.query.proximity.ProximityQuery -import org.vitrivr.engine.core.model.retrievable.attributes.DistanceAttribute -import org.vitrivr.engine.core.model.types.Value -import org.vitrivr.engine.core.operators.retrieve.Retriever - -/** - * [Retriever] implementation for the [HueHistogram] analyser. - * - * @see [HueHistogram] - * - * @author Ralph Gasser - * @version 1.0.0 - */ -class HueHistogramRetriever(field: Schema.Field, query: ProximityQuery, context: QueryContext) : AbstractRetriever(field, query, context) { - - companion object { - /** [LinearCorrespondence] for [HueHistogramRetriever]. Maximum distance is taken from Cineast implementation. */ - private val CORRESPONDENCE = LinearCorrespondence(16f) - } - - override fun toFlow(scope: CoroutineScope) = flow { - this@HueHistogramRetriever.reader.queryAndJoin(this@HueHistogramRetriever.query).forEach { - val distance = it.filteredAttribute() - if (distance != null) { - it.addAttribute(CORRESPONDENCE(distance)) - } else { - this@HueHistogramRetriever.logger.warn { "No distance attribute found for descriptor ${it.id}." } - } - emit(it) - } - } -} \ No newline at end of file diff --git a/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/dense/DenseEmbedding.kt b/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/dense/DenseEmbedding.kt index 2be8f709b..ee420a0eb 100644 --- a/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/dense/DenseEmbedding.kt +++ b/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/dense/DenseEmbedding.kt @@ -10,6 +10,7 @@ import org.vitrivr.engine.base.features.external.implementations.asr.ASRExtracto import org.vitrivr.engine.core.context.IndexContext import org.vitrivr.engine.core.context.QueryContext import org.vitrivr.engine.core.features.dense.DenseRetriever +import org.vitrivr.engine.core.math.correspondence.BoundedCorrespondence import org.vitrivr.engine.core.model.content.element.ContentElement import org.vitrivr.engine.core.model.content.element.ImageContent import org.vitrivr.engine.core.model.content.element.TextContent @@ -83,7 +84,7 @@ class DenseEmbedding : ExternalFesAnalyser, FloatVectorDescrip require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } require(query is ProximityQuery<*> && query.value is Value.FloatVector) { "The query is not a ProximityQuery." } @Suppress("UNCHECKED_CAST") - return DenseRetriever(field, query as ProximityQuery, context) + return DenseRetriever(field, query as ProximityQuery, context, BoundedCorrespondence(0.0f, 2.0f)) } /** From 7a23dc328a15b9d861e9035b31342be4c2f14c2e Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Wed, 21 Aug 2024 10:32:00 +0200 Subject: [PATCH 07/20] Added DominantColor and moved AverageColor to feature module --- .../descriptor/struct/LabelDescriptor.kt | 11 +- ...trivr.engine.core.model.metamodel.Analyser | 1 - .../feature}/averagecolor/AverageColor.kt | 2 +- .../averagecolor/AverageColorExtractor.kt | 2 +- .../averagecolor/AverageColorRetriever.kt | 2 +- .../feature/dominantcolor/DominantColor.kt | 158 ++++++++++++++++++ .../dominantcolor/DominantColorExtractor.kt | 34 ++++ .../dominantcolor/DominantColorRetriever.kt | 11 ++ ...trivr.engine.core.model.metamodel.Analyser | 4 +- 9 files changed, 219 insertions(+), 6 deletions(-) rename {vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features => vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature}/averagecolor/AverageColor.kt (99%) rename {vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features => vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature}/averagecolor/AverageColorExtractor.kt (96%) rename {vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features => vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature}/averagecolor/AverageColorRetriever.kt (96%) create mode 100644 vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/dominantcolor/DominantColor.kt create mode 100644 vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/dominantcolor/DominantColorExtractor.kt create mode 100644 vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/dominantcolor/DominantColorRetriever.kt diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/LabelDescriptor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/LabelDescriptor.kt index 15d42437e..0e1325545 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/LabelDescriptor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/LabelDescriptor.kt @@ -27,6 +27,14 @@ class LabelDescriptor( ) } + constructor( + id: DescriptorId, + retrievableId: RetrievableId?, + label: String, + confidence: Float = 1f, + field: Schema.Field<*, LabelDescriptor>? = null + ) : this(id, retrievableId, mapOf("label" to Value.String(label), "confidence" to Value.Float(confidence)), field) + /** The stored label. */ val label: Value.String by this.values @@ -41,5 +49,6 @@ class LabelDescriptor( * @param field [Schema.Field] the new [LabelDescriptor] belongs to. * @return Copy of this [LabelDescriptor]. */ - override fun copy(id: DescriptorId, retrievableId: RetrievableId?, field: Schema.Field<*, LabelDescriptor>?) = LabelDescriptor(id, retrievableId, HashMap(this.values), this.field) + override fun copy(id: DescriptorId, retrievableId: RetrievableId?, field: Schema.Field<*, LabelDescriptor>?) = + LabelDescriptor(id, retrievableId, HashMap(this.values), this.field) } diff --git a/vitrivr-engine-core/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser b/vitrivr-engine-core/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser index f3d140936..e9d29c3eb 100644 --- a/vitrivr-engine-core/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser +++ b/vitrivr-engine-core/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser @@ -1,4 +1,3 @@ -org.vitrivr.engine.core.features.averagecolor.AverageColor org.vitrivr.engine.core.features.metadata.source.file.FileSourceMetadata org.vitrivr.engine.core.features.metadata.source.exif.ExifMetadata org.vitrivr.engine.core.features.metadata.source.video.VideoSourceMetadata diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColor.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/averagecolor/AverageColor.kt similarity index 99% rename from vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColor.kt rename to vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/averagecolor/AverageColor.kt index 4b0f52c8d..820627fba 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColor.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/averagecolor/AverageColor.kt @@ -1,4 +1,4 @@ -package org.vitrivr.engine.core.features.averagecolor +package org.vitrivr.engine.module.features.feature.averagecolor import org.vitrivr.engine.core.context.IndexContext import org.vitrivr.engine.core.context.QueryContext diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColorExtractor.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/averagecolor/AverageColorExtractor.kt similarity index 96% rename from vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColorExtractor.kt rename to vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/averagecolor/AverageColorExtractor.kt index b580a3012..6392bec1f 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColorExtractor.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/averagecolor/AverageColorExtractor.kt @@ -1,4 +1,4 @@ -package org.vitrivr.engine.core.features.averagecolor +package org.vitrivr.engine.module.features.feature.averagecolor import org.vitrivr.engine.core.features.AbstractExtractor import org.vitrivr.engine.core.features.metadata.source.file.FileSourceMetadataExtractor diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColorRetriever.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/averagecolor/AverageColorRetriever.kt similarity index 96% rename from vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColorRetriever.kt rename to vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/averagecolor/AverageColorRetriever.kt index 126640442..84a0db406 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColorRetriever.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/averagecolor/AverageColorRetriever.kt @@ -1,4 +1,4 @@ -package org.vitrivr.engine.core.features.averagecolor +package org.vitrivr.engine.module.features.feature.averagecolor import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.flow.flow diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/dominantcolor/DominantColor.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/dominantcolor/DominantColor.kt new file mode 100644 index 000000000..f44c86320 --- /dev/null +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/dominantcolor/DominantColor.kt @@ -0,0 +1,158 @@ +package org.vitrivr.engine.module.features.feature.dominantcolor + +import org.vitrivr.engine.core.context.IndexContext +import org.vitrivr.engine.core.context.QueryContext +import org.vitrivr.engine.module.features.feature.averagecolor.AverageColorExtractor +import org.vitrivr.engine.core.model.color.ColorConverter +import org.vitrivr.engine.core.model.color.rgb.RGBByteColorContainer +import org.vitrivr.engine.core.model.content.element.ImageContent +import org.vitrivr.engine.core.model.descriptor.struct.LabelDescriptor +import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor +import org.vitrivr.engine.core.model.metamodel.Analyser +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.query.Query +import org.vitrivr.engine.core.model.query.bool.BooleanQuery +import org.vitrivr.engine.core.model.query.bool.SimpleBooleanQuery +import org.vitrivr.engine.core.model.query.proximity.ProximityQuery +import org.vitrivr.engine.core.model.retrievable.Retrievable +import org.vitrivr.engine.core.model.types.Value +import org.vitrivr.engine.core.operators.Operator +import org.vitrivr.engine.core.operators.ingest.Extractor +import org.vitrivr.engine.core.operators.retrieve.Retriever +import org.vitrivr.engine.core.util.extension.getRGBArray +import java.util.* + +class DominantColor : Analyser { + override val contentClasses = setOf(ImageContent::class) + override val descriptorClass = LabelDescriptor::class + + enum class ColorLabel { + UNDETERMINED, + BLACKWHITE, + RED, + ORANGE, + YELLOW, + GREEN, + CYAN, + BLUE, + VIOLET, + MAGENTA, + GRAY + } + + /** + * Generates a prototypical [FloatVectorDescriptor] for this [DominantColor]. + * + * @param field [Schema.Field] to create the prototype for. + * @return [FloatVectorDescriptor] + */ + override fun prototype(field: Schema.Field<*, *>) = LabelDescriptor(UUID.randomUUID(), UUID.randomUUID(), "") + + override fun newRetrieverForQuery( + field: Schema.Field, + query: Query, + context: QueryContext + ): Retriever { + + require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } + require(query is BooleanQuery) { "The query is not a BooleanQuery." } + + return DominantColorRetriever(field, query, context) + + } + + override fun newRetrieverForDescriptors( + field: Schema.Field, + descriptors: Collection, + context: QueryContext + ): Retriever { + + val labels = descriptors.mapNotNull { + try { + ColorLabel.valueOf(it.label.value.uppercase()) + } catch (e: IllegalArgumentException) { + null + } + }.toSet().map { it.name } + + val query = SimpleBooleanQuery(Value.String(labels.first()), attributeName = "label") + + return newRetrieverForQuery(field, query, context) + } + + override fun newRetrieverForContent( + field: Schema.Field, + content: Collection, + context: QueryContext + ): Retriever = newRetrieverForDescriptors(field, analyse(content), context) + + + + override fun newExtractor( + field: Schema.Field, + input: Operator, + context: IndexContext + ): Extractor = DominantColorExtractor(input, this, field) + + override fun newExtractor( + name: String, + input: Operator, + context: IndexContext + ): Extractor = DominantColorExtractor(input, this, null) //TODO name + + /** + * Performs the [DominantColor] analysis on the provided [List] of [ImageContent] elements. + * Adapted from Cineast. + * + * @param content The [List] of [ImageContent] elements. + * @return [List] of [LabelDescriptor]s. + */ + fun analyse(content: Collection): List = content.map { + val hist = IntArray(10) { 0 } + val rgb = it.content.getRGBArray() + rgb.forEach { color -> + val hsv = ColorConverter.rgbToHsv(RGBByteColorContainer(color)) + + if(hsv.saturation < 0.02f){ + ++hist[0]; + return@forEach; + } + if(hsv.saturation < 0.2f || hsv.value < 0.3f){ + ++hist[9]; + return@forEach; + } + else if(hsv.hue >= 0.07 && hsv.hue < 0.14){ //orange + ++hist[2]; + } + else if(hsv.hue >= 0.14 && hsv.hue < 0.17){ //yellow + ++hist[3]; + } + else if(hsv.hue >= 0.17 && hsv.hue < 0.44){ //green + ++hist[4]; + } + else if(hsv.hue >= 0.44 && hsv.hue < 0.56){ //cyan + ++hist[5]; + } + else if(hsv.hue >= 0.56 && hsv.hue < 0.73){ //blue + ++hist[6]; + } + else if(hsv.hue >= 0.73 && hsv.hue < 0.76){ //violet + ++hist[7]; + } + else if(hsv.hue >= 0.76 && hsv.hue < 0.92){ //magenta + ++hist[8]; + } + else{ + ++hist[1]; + } + + } + + val max = hist.max() + val label = if (max < rgb.size / 2) ColorLabel.UNDETERMINED else ColorLabel.entries[hist.indexOf(max)] + + return@map LabelDescriptor(UUID.randomUUID(), UUID.randomUUID(), label.name) + + } + +} \ No newline at end of file diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/dominantcolor/DominantColorExtractor.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/dominantcolor/DominantColorExtractor.kt new file mode 100644 index 000000000..69743a151 --- /dev/null +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/dominantcolor/DominantColorExtractor.kt @@ -0,0 +1,34 @@ +package org.vitrivr.engine.module.features.feature.dominantcolor + +import org.vitrivr.engine.core.features.AbstractExtractor +import org.vitrivr.engine.core.features.metadata.source.file.FileSourceMetadataExtractor +import org.vitrivr.engine.core.model.content.ContentType +import org.vitrivr.engine.core.model.content.element.ImageContent +import org.vitrivr.engine.core.model.descriptor.struct.LabelDescriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.retrievable.Retrievable +import org.vitrivr.engine.core.operators.Operator +import org.vitrivr.engine.core.operators.ingest.Extractor +import org.vitrivr.engine.core.source.file.FileSource + +class DominantColorExtractor(input: Operator, analyser: DominantColor, field: Schema.Field?) : AbstractExtractor(input, analyser, field) { + + /** + * Internal method to check, if [Retrievable] matches this [Extractor] and should thus be processed. + * + * [FileSourceMetadataExtractor] implementation only works with [Retrievable] that contain a [FileSource]. + * + * @param retrievable The [Retrievable] to check. + * @return True on match, false otherwise, + */ + override fun matches(retrievable: Retrievable): Boolean = retrievable.content.any { it.type == ContentType.BITMAP_IMAGE } + + + + override fun extract(retrievable: Retrievable): List { + val content = retrievable.content.filterIsInstance() + return (this.analyser as DominantColor).analyse(content).map { it.copy(retrievableId = retrievable.id, field = this@DominantColorExtractor.field) } + } + + +} \ No newline at end of file diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/dominantcolor/DominantColorRetriever.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/dominantcolor/DominantColorRetriever.kt new file mode 100644 index 000000000..8dce1bba5 --- /dev/null +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/dominantcolor/DominantColorRetriever.kt @@ -0,0 +1,11 @@ +package org.vitrivr.engine.module.features.feature.dominantcolor + +import org.vitrivr.engine.core.context.QueryContext +import org.vitrivr.engine.core.features.AbstractRetriever +import org.vitrivr.engine.core.model.content.element.ImageContent +import org.vitrivr.engine.core.model.descriptor.struct.LabelDescriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.query.bool.BooleanQuery + + +class DominantColorRetriever(field: Schema.Field, query: BooleanQuery, context: QueryContext) : AbstractRetriever(field, query, context) \ No newline at end of file diff --git a/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser b/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser index 032bde4d2..8a541238f 100644 --- a/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser +++ b/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser @@ -1,4 +1,6 @@ org.vitrivr.engine.module.features.feature.ehd.EHD org.vitrivr.engine.module.features.feature.huehistogram.HueHistogram org.vitrivr.engine.module.features.feature.external.implementations.dino.DINO -org.vitrivr.engine.module.features.feature.external.implementations.clip.CLIP \ No newline at end of file +org.vitrivr.engine.module.features.feature.external.implementations.clip.CLIP +org.vitrivr.engine.module.features.feature.averagecolor.AverageColor +org.vitrivr.engine.module.features.feature.dominantcolor.DominantColor From a0d86e738938004cf2fb6f8828b678c8fe2a0712 Mon Sep 17 00:00:00 2001 From: Ralph Gasser Date: Wed, 21 Aug 2024 11:29:24 +0200 Subject: [PATCH 08/20] Simplifies different types of color container and adds LabColorContainer and XYZColorContainer. Signed-off-by: Ralph Gasser --- .../features/averagecolor/AverageColor.kt | 15 +- .../engine/core/model/color/ColorConverter.kt | 44 ----- .../core/model/color/HSVColorContainer.kt | 99 ++++++++++ .../core/model/color/LabColorContainer.kt | 37 ++++ .../core/model/color/RGBColorContainer.kt | 170 ++++++++++++++++++ .../core/model/color/XYZColorContainer.kt | 106 +++++++++++ .../core/model/color/hsv/HSVColorContainer.kt | 26 --- .../rgb/MutableRGBFloatColorContainer.kt | 105 ----------- .../model/color/rgb/RGBByteColorContainer.kt | 47 ----- .../model/color/rgb/RGBFloatColorContainer.kt | 99 ---------- .../engine/core/util/math/StatisticsHelper.kt | 31 ++++ .../image/AverageImageContentAggregator.kt | 17 +- .../RepresentativeImageContentAggregator.kt | 19 +- .../feature/huehistogram/HueHistogram.kt | 7 +- 14 files changed, 478 insertions(+), 344 deletions(-) delete mode 100644 vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/ColorConverter.kt create mode 100644 vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/HSVColorContainer.kt create mode 100644 vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/LabColorContainer.kt create mode 100644 vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBColorContainer.kt create mode 100644 vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/XYZColorContainer.kt delete mode 100644 vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/hsv/HSVColorContainer.kt delete mode 100644 vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/rgb/MutableRGBFloatColorContainer.kt delete mode 100644 vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/rgb/RGBByteColorContainer.kt delete mode 100644 vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/rgb/RGBFloatColorContainer.kt create mode 100644 vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/util/math/StatisticsHelper.kt diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColor.kt index cedafc42f..17b16c4c2 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColor.kt @@ -4,9 +4,7 @@ import org.vitrivr.engine.core.context.IndexContext import org.vitrivr.engine.core.context.QueryContext import org.vitrivr.engine.core.features.dense.DenseRetriever import org.vitrivr.engine.core.math.correspondence.LinearCorrespondence -import org.vitrivr.engine.core.model.color.rgb.MutableRGBFloatColorContainer -import org.vitrivr.engine.core.model.color.rgb.RGBByteColorContainer -import org.vitrivr.engine.core.model.color.rgb.RGBFloatColorContainer +import org.vitrivr.engine.core.model.color.RGBColorContainer import org.vitrivr.engine.core.model.content.Content import org.vitrivr.engine.core.model.content.element.ImageContent import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor @@ -120,12 +118,17 @@ class AverageColor : Analyser { * @return [List] of [FloatVectorDescriptor]s. */ fun analyse(content: Collection): List = content.map { - val color = MutableRGBFloatColorContainer() + val color = floatArrayOf(0f, 0f, 0f) val rgb = it.content.getRGBArray() - rgb.forEach { c -> color += RGBByteColorContainer(c) } + for (c in rgb) { + val container = RGBColorContainer(c) + color[0] += container.red + color[1] += container.green + color[2] += container.blue + } /* Generate descriptor. */ - val averageColor = RGBFloatColorContainer(color.red / rgb.size, color.green / rgb.size, color.blue / rgb.size) + val averageColor = RGBColorContainer(color[0] / rgb.size, color[1] / rgb.size, color[2] / rgb.size) FloatVectorDescriptor(UUID.randomUUID(), null, averageColor.toVector()) } } diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/ColorConverter.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/ColorConverter.kt deleted file mode 100644 index 871019e69..000000000 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/ColorConverter.kt +++ /dev/null @@ -1,44 +0,0 @@ -package org.vitrivr.engine.core.model.color - -import org.vitrivr.engine.core.model.color.hsv.HSVColorContainer -import org.vitrivr.engine.core.model.color.rgb.RGBByteColorContainer -import org.vitrivr.engine.core.model.color.rgb.RGBFloatColorContainer - -/** - * A color converter that provides methods to convert between different color spaces. - * - * Adapted from Cineast. - * - * @author Ralph Gasser - * @version 1.0.0 - */ -object ColorConverter { - - /** - * Converts a [RGBByteColorContainer] to a [HSVColorContainer]. - * - * @param rgb The [RGBByteColorContainer] to convert. - * @return [HSVColorContainer] - */ - fun rgbToHsv(rgb: RGBByteColorContainer): HSVColorContainer = rgbToHsv(rgb.toFloatContainer()) - - /** - * Converts a [RGBFloatColorContainer] to a [HSVColorContainer]. - * - * @param rgb The [RGBFloatColorContainer] to convert. - * @return [HSVColorContainer] - */ - fun rgbToHsv(rgb: RGBFloatColorContainer): HSVColorContainer { - val max = maxOf(rgb.red, rgb.green, rgb.blue) - val min = minOf(rgb.red, rgb.green, rgb.blue) - val d = max - min - val s = if (max == 0f) 0f else d / max - val h = when { - d == 0f -> 0f - max == rgb.red -> (rgb.green - rgb.blue) / d + (if (rgb.green < rgb.blue) 6 else 0) - max == rgb.green -> (rgb.blue - rgb.red) / d + 2 - else -> (rgb.red - rgb.green) / d + 4 - } - return HSVColorContainer(h, s, max) - } -} \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/HSVColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/HSVColorContainer.kt new file mode 100644 index 000000000..54e00e146 --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/HSVColorContainer.kt @@ -0,0 +1,99 @@ +package org.vitrivr.engine.core.model.color + +import org.vitrivr.engine.core.model.types.Value +import kotlin.math.floor + +/** + * A container for HSV colors. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +@JvmInline +value class HSVColorContainer constructor(private val colors: FloatArray) { + + constructor(hue: Float, saturation: Float, value: Float) : this(floatArrayOf(hue, saturation, value)) + constructor(hue: Double, saturation: Double, value: Double) : this(floatArrayOf(hue.toFloat(), saturation.toFloat(), value.toFloat())) + + /** Accessor for the hue component of the [HSVColorContainer]. */ + val hue: Float + get() = this.colors[0] + + /** Accessor for the saturation component of the [HSVColorContainer]. */ + val saturation: Float + get() = this.colors[1] + + /** Accessor for the value component of the [HSVColorContainer]. */ + val value: Float + get() = this.colors[2] + + /** + * Converts this [HSVColorContainer] a [Value.FloatVector] + * + * @return [Value.FloatVector] + */ + fun toVector() = Value.FloatVector(this.colors) + + /** + * Converts this [HSVColorContainer] to a [RGBColorContainer]. + * + * @return [RGBColorContainer] + */ + fun toRGB(): RGBColorContainer { + var r = 0 + var g = 0 + var b = 0 + if (this.saturation < 0.00001f) { + b = (this.value * 255f).toInt() + g = b + r = g + } else { + val h: Double = (this.hue * 6.0) % 6.0 + val i = floor(h).toInt() + val v1 = (this.value * (1.0 - this.saturation) * 255f).toInt() + val v2 = (this.value * (1.0 - this.saturation * (h - i)) * 255f).toInt() + val v3 = (this.value * (1.0 - this.saturation * (1 - (h - i))) * 255f).toInt() + + when (i) { + 0 -> { + r = (255f * this.value).toInt() + g = v3 + b = v1 + } + + 1 -> { + r = v2 + g = (255f * this.value).toInt() + b = v1 + } + + 2 -> { + r = v1 + g = (255f * this.value).toInt() + b = v3 + } + + 3 -> { + r = v1 + g = v2 + b = (this.value * 255f).toInt() + } + + 4 -> { + r = v3 + g = v1 + b = (this.value * 255f).toInt() + } + + else -> { + r = (255f * this.value).toInt() + g = v1 + b = v2 + } + } + } + return RGBColorContainer(r, g, b) + } + + override fun toString(): String = "HSVFloatColorContainer(H=$hue, S=$saturation, V=$value)" +} \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/LabColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/LabColorContainer.kt new file mode 100644 index 000000000..3e1c194a2 --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/LabColorContainer.kt @@ -0,0 +1,37 @@ +package org.vitrivr.engine.core.model.color + +import org.vitrivr.engine.core.model.types.Value + +/** + * A container for LAB colors. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +@JvmInline +value class LabColorContainer constructor(private val colors: FloatArray) { + + constructor(l: Float, a: Float, b: Float) : this(floatArrayOf(l, a, b)) + constructor(l: Double, a: Double, b: Double) : this(floatArrayOf(l.toFloat(), a.toFloat(), b.toFloat())) + + /** Accessor for the L component of the [LabColorContainer]. */ + val l: Float + get() = this.colors[0] + + /** Accessor for the A component of the [LabColorContainer]. */ + val a: Float + get() = this.colors[1] + + /** Accessor for the B component of the [LabColorContainer]. */ + val b: Float + get() = this.colors[2] + + /** + * Converts this [LabColorContainer] a [Value.FloatVector] + * + * @return [Value.FloatVector] + */ + fun toVector() = Value.FloatVector(this.colors) + + override fun toString(): String = "LabFloatColorContainer(L=$l, A=$a, B=$b)" +} \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBColorContainer.kt new file mode 100644 index 000000000..a6b767b57 --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBColorContainer.kt @@ -0,0 +1,170 @@ +package org.vitrivr.engine.core.model.color + +import org.vitrivr.engine.core.model.types.Value +import kotlin.math.max +import kotlin.math.min +import kotlin.math.pow +import kotlin.math.sqrt + +/** + * A container for RGB colors. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +@JvmInline +value class RGBColorContainer constructor(private val rgb: FloatArray) { + + init { + require(this.rgb.size == 3) { "RGBFloatColorContainer must have exactly 3 elements." } + require(this.rgb[0] in 0.0f..1.0f) { "RGBFloatColorContainer components must be between 0.0 and 1.0." } + require(this.rgb[1] in 0.0f..1.0f) { "RGBFloatColorContainer components must be between 0.0 and 1.0." } + require(this.rgb[2] in 0.0f..1.0f) { "RGBFloatColorContainer components must be between 0.0 and 1.0." } + } + + constructor(rgb: Int) : this((rgb shr 16 and 0xFF).toByte(), (rgb shr 8 and 0xFF).toByte(), (rgb and 0xFF).toByte()) + constructor(red: Byte, green: Byte, blue: Byte) : this(red.toFloat() / 255f, green.toFloat() / 255f, blue.toFloat() / 255f) + constructor(red: Int, green: Int, blue: Int) : this(red.toFloat() / 255f, green.toFloat() / 255f, blue.toFloat() / 255f) + constructor(r: Double, g: Double, b: Double) : this(r.toFloat(), g.toFloat(), b.toFloat()) + constructor(red: Float, green: Float, blue: Float) : this(floatArrayOf(red, green, blue)) + + /** Accessor for the red component of the [RGBColorContainer]. */ + val red: Float + get() = this.rgb[0] + + /** Accessor for the green component of the [RGBColorContainer]. */ + val green: Float + get() = this.rgb[1] + + /** Accessor for the blue component of the [RGBColorContainer]. */ + val blue: Float + get() = this.rgb[2] + + /** + * Adds this [RGBColorContainer] to another [RGBColorContainer]. + * + * @param other [RGBColorContainer] to add. + * @return [RGBColorContainer] + */ + operator fun plus(other: RGBColorContainer): RGBColorContainer = + RGBColorContainer(this.red + other.red, this.green + other.green, this.blue + other.blue) + + /** + * Subtracts a [RGBColorContainer] from this [RGBColorContainer]. + * + * @param other [RGBColorContainer] to subtract. + * @return [RGBColorContainer] + */ + operator fun minus(other: RGBColorContainer): RGBColorContainer = RGBColorContainer( + this.red - other.red, + this.green - other.green, + this.blue - other.blue + ) + + /** + * Clamps the values of each color channel to [0f, 1f] + */ + fun clamped(): RGBColorContainer = RGBColorContainer(min(1f, max(0f, this.red)), min(1f, max(0f, this.green)), min(1f, max(0f, this.blue))) + + /** + * Normalizes the magnitude of the color vector to 1 + */ + fun normalized(): RGBColorContainer { + val magnitude = sqrt(this.red * this.red + this.green * this.green + this.blue * this.blue) + return if (magnitude > 0f) { + RGBColorContainer(this.red / magnitude, this.green / magnitude, this.blue / magnitude) + } else { + this + } + } + + /** + * Converts this [RGBColorContainer] to an [Int] representation. + * + * @return [Int] representation of RGB color + */ + fun toRGBInt(): Int = (this.blue * 255).toInt() and 0XFF or ((this.green * 255).toInt() and 0xFF shl 8) or (((this.red * 255).toInt() and 0xFF) shl 16) + + /** + * Converts this [RGBColorContainer] to a [HSVColorContainer]. + * + * @return [HSVColorContainer] + */ + fun toHSV(): HSVColorContainer { + val max = maxOf(this.red, this.green, this.blue) + val min = minOf(this.red, this.green, this.blue) + val d = max - min + val s = if (max == 0f) 0f else d / max + val h = when { + d == 0f -> 0f + max == this.red -> (this.green - this.blue) / d + (if (this.green < this.blue) 6 else 0) + max == this.green -> (this.blue - this.red) / d + 2 + else -> (this.red - this.green) / d + 4 + } + return HSVColorContainer(h, s, max) + } + + /** + * Converts this [RGBColorContainer] to a [XYZColorContainer]. + * + * @return [XYZColorContainer] + */ + fun toXYZ(): XYZColorContainer { + var r = this.red.toDouble() + var g = this.green.toDouble() + var b = this.blue.toDouble() + if (r > 0.04045) { + r = ((r + 0.055) / 1.055).pow(2.4) + } else { + r /= 12.92 + } + + if (g > 0.04045) { + g = ((g + 0.055) / 1.055).pow(2.4) + } else { + g /= 12.92 + } + + if (b > 0.04045) { + b = ((b + 0.055) / 1.055).pow(2.4) + } else { + b /= 12.92 + } + + r *= 100.0 + g *= 100.0 + b *= 100.0 + + return XYZColorContainer( + r * 0.4121 + g * 0.3576 + b * 0.1805, + r * 0.2126 + g * 0.7152 + b * 0.0722, + r * 0.0193 + g * 0.1192 + b * 0.9505 + ) + } + + /** + * Converts this [RGBColorContainer] as a [List] of [Float]s. + * + * @return [List] of [Float]s + */ + fun toList() = listOf(this.red, this.green, this.blue) + + /** + * Converts this [RGBColorContainer] a [Value.FloatVector] + * + * @return [Value.FloatVector] + */ + fun toVector() = Value.FloatVector(this.rgb) + + /** + * Calculates distance to another [RGBColorContainer]. + * + * @param other [RGBColorContainer] to calculate distance to. + * @return Distance to [other]. + */ + fun distanceTo(other: RGBColorContainer): Float = sqrt( + (this.red - other.red) * (this.red - other.red) + (this.green - other.green) * (this.green - other.green) + (this.blue - other.blue) * (this.blue - other.blue) + ) + + override fun toString(): String = "RGBFloatColorContainer(R=$red, G=$green, B=$blue)" +} diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/XYZColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/XYZColorContainer.kt new file mode 100644 index 000000000..27591c1da --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/XYZColorContainer.kt @@ -0,0 +1,106 @@ +package org.vitrivr.engine.core.model.color + +import org.vitrivr.engine.core.model.types.Value +import kotlin.math.pow + +/** + * A container for XYZ colors. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +@JvmInline +value class XYZColorContainer constructor(private val colors: FloatArray) { + + constructor(x: Float, y: Float, z: Float) : this(floatArrayOf(x, y, z)) + constructor(x: Double, y: Double, z: Double) : this(floatArrayOf(x.toFloat(), y.toFloat(), z.toFloat())) + + /** Accessor for the L component of the [LabColorContainer]. */ + val x: Float + get() = this.colors[0] + + /** Accessor for the A component of the [LabColorContainer]. */ + val y: Float + get() = this.colors[1] + + /** Accessor for the B component of the [LabColorContainer]. */ + val z: Float + get() = this.colors[2] + + /** + * Converts this [XYZColorContainer] to a [LabColorContainer]. + * + * @return [LabColorContainer] + */ + fun toLab(): LabColorContainer { + var x = this.x / 95.047 + var y = this.y / 100.0 + var z = this.z / 108.883 + + if (x > 0.008856f) { + x = x.pow((1.0 / 3.0)) + } else { + x = (x * 7.787 + (16.0 / 116.0)) + } + + y = if (y > 0.008856f) { + y.pow((1.0 / 3.0)) + } else { + (y * 7.787 + (16.0 / 116.0)) + } + + z = if (z > 0.008856f) { + z.pow((1.0 / 3.0)) + } else { + (z * 7.787 + (16.0 / 116.0)) + } + + return LabColorContainer((116.0 * y) - 16.0, 500.0 * (x - y), 200.0 * (y - z)) + } + + /** + * Converts this [XYZColorContainer] to a [RGBColorContainer]. + * + * @return [LabColorContainer] + */ + fun toRGB(): RGBColorContainer { + val x: Double = this.x / 100.0 + val y: Double = this.y / 100.0 + val z: Double = this.z / 100.0 + + var r = x * 3.2406 + y * -1.5372 + z * -0.4986 + var g = x * -0.9689 + y * 1.8758 + z * 0.0415 + var b = x * 0.0557 + y * -0.2040 + z * 1.0570 + + if (r > 0.0031308) { + r = 1.055 * r.pow((1.0 / 2.4)) - 0.055 + } else { + r *= 12.92 + } + + if (g > 0.0031308) { + g = 1.055 * g.pow((1.0 / 2.4)) - 0.055 + } else { + g *= 12.92 + } + + if (b > 0.0031308) { + b = 1.055 * b.pow((1.0 / 2.4)) - 0.055 + } else { + b *= 12.92 + } + + return RGBColorContainer(r, g, b) + } + + /** + * Converts this [XYZColorContainer] a [Value.FloatVector] + * + * @return [Value.FloatVector] + */ + fun toVector() = Value.FloatVector(this.colors) + + override fun toString(): String { + return "XYZFloatColorContainer(X=$x, Y=$y, Z=$z)" + } +} \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/hsv/HSVColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/hsv/HSVColorContainer.kt deleted file mode 100644 index 00b5f0761..000000000 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/hsv/HSVColorContainer.kt +++ /dev/null @@ -1,26 +0,0 @@ -package org.vitrivr.engine.core.model.color.hsv - -/** - * A container for HSV colors. - * - * @author Ralph Gasser - * @version 1.0.0 - */ -@JvmInline -value class HSVColorContainer private constructor(private val colors: FloatArray) { - - constructor(hue: Float, saturation: Float, value: Float) : this(floatArrayOf(hue, saturation, value)) - constructor(hue: Double, saturation: Double, value: Double) : this(floatArrayOf(hue.toFloat(), saturation.toFloat(), value.toFloat())) - - /** Accessor for the hue component of the [HSVColorContainer]. */ - val hue: Float - get() = this.colors[0] - - /** Accessor for the saturation component of the [HSVColorContainer]. */ - val saturation: Float - get() = this.colors[1] - - /** Accessor for the value component of the [HSVColorContainer]. */ - val value: Float - get() = this.colors[2] -} \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/rgb/MutableRGBFloatColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/rgb/MutableRGBFloatColorContainer.kt deleted file mode 100644 index aa5eb78a2..000000000 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/rgb/MutableRGBFloatColorContainer.kt +++ /dev/null @@ -1,105 +0,0 @@ -package org.vitrivr.engine.core.model.color.rgb - -import kotlin.math.max -import kotlin.math.min -import kotlin.math.sqrt - -data class MutableRGBFloatColorContainer(override var red: Float, override var green: Float, override var blue: Float) : - RGBFloatColorContainer(red, green, blue) { - - constructor() : this(0f, 0f, 0f) - - override operator fun plus(other: RGBFloatColorContainer): MutableRGBFloatColorContainer = - MutableRGBFloatColorContainer(this.red + other.red, this.green + other.green, this.blue + other.blue) - - override operator fun plus(other: RGBByteColorContainer): MutableRGBFloatColorContainer = - MutableRGBFloatColorContainer( - this.red + (other.red.toFloat() / 255f), - this.green + (other.green.toFloat() / 255f), - this.blue + (other.blue.toFloat() / 255f) - ) - - operator fun plusAssign(other: RGBFloatColorContainer) { - this.red += other.red - this.green += other.green - this.blue += other.blue - } - - operator fun plusAssign(other: RGBByteColorContainer) { - this.red += other.red.toFloat() / 255f - this.green += other.green.toFloat() / 255f - this.blue += other.blue.toFloat() / 255f - } - - override operator fun minus(other: RGBFloatColorContainer): MutableRGBFloatColorContainer = - MutableRGBFloatColorContainer(this.red - other.red, this.green - other.green, this.blue - other.blue) - - override operator fun minus(other: RGBByteColorContainer): MutableRGBFloatColorContainer = - MutableRGBFloatColorContainer( - this.red - (other.red.toFloat() / 255f), - this.green - (other.green.toFloat() / 255f), - this.blue - (other.blue.toFloat() / 255f) - ) - - operator fun times(value: Float) : MutableRGBFloatColorContainer = MutableRGBFloatColorContainer(this.red * value, this.green * value, this.blue * value) - - operator fun timesAssign(value: Float) { - this.red *= value - this.green *= value - this.blue *= value - } - - operator fun div(value: Float) : MutableRGBFloatColorContainer = MutableRGBFloatColorContainer(this.red / value, this.green / value, this.blue / value) - - operator fun divAssign(value: Float) { - this.red /= value - this.green /= value - this.blue /= value - } - - /** - * Clamps the values of each color channel to [0f, 1f] - */ - override fun clamped(): MutableRGBFloatColorContainer = - MutableRGBFloatColorContainer( - min(1f, max(0f, this.red)), - min(1f, max(0f, this.green)), - min(1f, max(0f, this.blue)) - ) - - /** - * Clamps the values of each color channel to [0f, 1f] in this instance - */ - fun clamp(): MutableRGBFloatColorContainer { - this.red = min(1f, max(0f, this.red)) - this.green = min(1f, max(0f, this.green)) - this.blue = min(1f, max(0f, this.blue)) - return this - } - - /** - * Normalizes the magnitude of the color vector to 1 - */ - override fun normalized(): MutableRGBFloatColorContainer { - val magnitude = sqrt(this.red * this.red + this.green * this.green + this.blue * this.blue) - return if (magnitude > 0f) { - MutableRGBFloatColorContainer(this.red / magnitude, this.green / magnitude, this.blue / magnitude) - } else { - this - } - } - - /** - * Normalizes the magnitude of this color vector to 1 - */ - fun normalize(): MutableRGBFloatColorContainer { - val magnitude = sqrt(this.red * this.red + this.green * this.green + this.blue * this.blue) - if (magnitude > 0f) { - this.red /= magnitude - this.green /= magnitude - this.blue /= magnitude - } - return this - } - -} diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/rgb/RGBByteColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/rgb/RGBByteColorContainer.kt deleted file mode 100644 index dfe77f15e..000000000 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/rgb/RGBByteColorContainer.kt +++ /dev/null @@ -1,47 +0,0 @@ -package org.vitrivr.engine.core.model.color.rgb - -/** - * A container for RGB colors. - * - * @version 1.1.0 - * @author Luca Rossetto - * @author Ralph Gasser - */ -@JvmInline -value class RGBByteColorContainer(private val rgb: Int) { - /** - * Constructor for creating a [RGBByteColorContainer] from RGB [UByte] values. - * - * @param red Red component. - * @param green Green component. - * @param blue Blue component. - */ - constructor(red: UByte, green: UByte, blue: UByte) : this(blue.toInt() and 0xFF or ((green.toInt() and 0xFF) shl 8) or ((red.toInt() and 0xFF) shl 16)) - - /** Red component of this [RGBByteColorContainer]. */ - val red: UByte - get() = (this.rgb shr 16 and 0xFF).toUByte() - - /** Green component of this [RGBByteColorContainer]. */ - val green: UByte - get() = (this.rgb shr 8 and 0xFF).toUByte() - - /** Blue component of this [RGBByteColorContainer]. */ - val blue: UByte - get() = (this.rgb and 0xFF).toUByte() - - /** - * Converts this [RGBByteColorContainer] to an [RGBFloatColorContainer]. - * - * @return [RGBFloatColorContainer] representation of this [RGBByteColorContainer]. - */ - fun toFloatContainer(): RGBFloatColorContainer = - RGBFloatColorContainer(this.red.toFloat() / 255f, this.green.toFloat() / 255f, this.blue.toFloat() / 255f) - - /** - * Converts this [RGBByteColorContainer] to an [Int] representation. - * - * @return [Int] representation of this [RGBByteColorContainer]. - */ - fun toRGBInt(): Int = this.rgb -} diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/rgb/RGBFloatColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/rgb/RGBFloatColorContainer.kt deleted file mode 100644 index a478f6086..000000000 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/rgb/RGBFloatColorContainer.kt +++ /dev/null @@ -1,99 +0,0 @@ -package org.vitrivr.engine.core.model.color.rgb - -import org.vitrivr.engine.core.model.types.Value -import kotlin.math.max -import kotlin.math.min -import kotlin.math.roundToInt -import kotlin.math.sqrt - -open class RGBFloatColorContainer(open val red: Float, open val green: Float, open val blue: Float) { - - constructor(floats: List) : this(floats[0], floats[1], floats[2]) - - open operator fun plus(other: RGBFloatColorContainer): RGBFloatColorContainer = - RGBFloatColorContainer(this.red + other.red, this.green + other.green, this.blue + other.blue) - - open operator fun plus(other: RGBByteColorContainer): RGBFloatColorContainer = RGBFloatColorContainer( - this.red + (other.red.toFloat() / 255f), - this.green + (other.green.toFloat() / 255f), - this.blue + (other.blue.toFloat() / 255f) - ) - - open operator fun minus(other: RGBFloatColorContainer): RGBFloatColorContainer = - RGBFloatColorContainer(this.red - other.red, this.green - other.green, this.blue - other.blue) - - open operator fun minus(other: RGBByteColorContainer): RGBFloatColorContainer = - RGBFloatColorContainer( - this.red - (other.red.toFloat() / 255f), - this.green - (other.green.toFloat() / 255f), - this.blue - (other.blue.toFloat() / 255f) - ) - - /** - * Clamps the values of each color channel to [0f, 1f] - */ - open fun clamped(): RGBFloatColorContainer = - RGBFloatColorContainer(min(1f, max(0f, this.red)), min(1f, max(0f, this.green)), min(1f, max(0f, this.blue))) - - /** - * Normalizes the magnitude of the color vector to 1 - */ - open fun normalized(): RGBFloatColorContainer { - val magnitude = sqrt(this.red * this.red + this.green * this.green + this.blue * this.blue) - return if (magnitude > 0f) { - RGBFloatColorContainer(this.red / magnitude, this.green / magnitude, this.blue / magnitude) - } else { - this - } - } - - fun toMutable(): MutableRGBFloatColorContainer = MutableRGBFloatColorContainer(this.red, this.green, this.blue) - - fun toByteContainer(): RGBByteColorContainer = RGBByteColorContainer( - (min(1f, max(0f, this.red)) * 255f).roundToInt().toUByte(), - (min(1f, max(0f, this.green)) * 255f).roundToInt().toUByte(), - (min(1f, max(0f, this.blue)) * 255f).roundToInt().toUByte() - ) - - /** - * Converts this [RGBFloatColorContainer] as a [List] of [Float]s. - * - * @return [List] of [Float]s - */ - fun toList() = listOf(this.red, this.green, this.blue) - - /** - * Converts this [RGBFloatColorContainer] as a [List] of [Value.Float]s. - * - * @return [List] of [Value.Float]s - */ - fun toVector() = Value.FloatVector(floatArrayOf(this.red, this.green, this.blue)) - - fun distanceTo(other: RGBFloatColorContainer): Float = sqrt( - (this.red - other.red) * (this.red - other.red) + (this.green - other.green) * (this.green - other.green) + (this.blue - other.blue) * (this.blue - other.blue) - ) - - override fun equals(other: Any?): Boolean { - if (this === other) return true - if (other !is RGBFloatColorContainer) return false - - if (red != other.red) return false - if (green != other.green) return false - if (blue != other.blue) return false - - return true - } - - override fun hashCode(): Int { - var result = red.hashCode() - result = 31 * result + green.hashCode() - result = 31 * result + blue.hashCode() - return result - } - - override fun toString(): String { - return "RGBFloatColorContainer(red=$red, green=$green, blue=$blue)" - } - - -} diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/util/math/StatisticsHelper.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/util/math/StatisticsHelper.kt new file mode 100644 index 000000000..9059f1af8 --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/util/math/StatisticsHelper.kt @@ -0,0 +1,31 @@ +package org.vitrivr.engine.core.util.math + +/** + * A collection of helper methods for basic statistics. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +object StatisticsHelper { + /** + * Extracts the median value from a [IntArray] histogram. + * + * @param hist The [IntArray] representing the histogram. + * @return The median value. + */ + fun medianFromHistogram(hist: IntArray): Int { + var posL = 0 + var posR = hist.size - 1 + var sumL = hist[posL] + var sumR = hist[posR] + + while (posL < posR) { + if (sumL < sumR) { + sumL += hist[++posL] + } else { + sumR += hist[--posR] + } + } + return posL + } +} \ No newline at end of file diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/AverageImageContentAggregator.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/AverageImageContentAggregator.kt index c857e3bda..9d56851f0 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/AverageImageContentAggregator.kt +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/AverageImageContentAggregator.kt @@ -2,8 +2,7 @@ package org.vitrivr.engine.index.aggregators.image import org.vitrivr.engine.core.context.Context import org.vitrivr.engine.core.context.IndexContext -import org.vitrivr.engine.core.model.color.rgb.MutableRGBFloatColorContainer -import org.vitrivr.engine.core.model.color.rgb.RGBByteColorContainer +import org.vitrivr.engine.core.model.color.RGBColorContainer import org.vitrivr.engine.core.model.content.decorators.SourcedContent import org.vitrivr.engine.core.model.content.decorators.TemporalContent import org.vitrivr.engine.core.model.content.element.ContentElement @@ -52,17 +51,23 @@ class AverageImageContentAggregator : TransformerFactory { val firstImage = images.first() val height = firstImage.height val width = firstImage.width - val colors = List(firstImage.width * firstImage.height) { MutableRGBFloatColorContainer() } + val colors = List(firstImage.width * firstImage.height) { floatArrayOf(0f, 0f, 0f) } images.forEach { imageContent -> require(imageContent.height == height && imageContent.width == width) { "Unable to aggregate images! All images must have same dimension." } imageContent.content.getRGBArray().forEachIndexed { index, color -> - colors[index] += RGBByteColorContainer(color) + val rgb = RGBColorContainer(color) + colors[index][0] += rgb.red + colors[index][1] += rgb.green + colors[index][2] += rgb.blue } } val div = images.size.toFloat() - val intColors = colors.map { - (it / div).toByteContainer().toRGBInt() + val intColors = colors.map { c -> + c[0] /= div + c[1] /= div + c[2] /= div + RGBColorContainer(c).toRGBInt() }.toIntArray() /* Prepare buffered image. */ diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/RepresentativeImageContentAggregator.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/RepresentativeImageContentAggregator.kt index e38d944da..f2218e52f 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/RepresentativeImageContentAggregator.kt +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/RepresentativeImageContentAggregator.kt @@ -2,8 +2,7 @@ package org.vitrivr.engine.index.aggregators.image import org.vitrivr.engine.core.context.Context import org.vitrivr.engine.core.context.IndexContext -import org.vitrivr.engine.core.model.color.rgb.MutableRGBFloatColorContainer -import org.vitrivr.engine.core.model.color.rgb.RGBByteColorContainer +import org.vitrivr.engine.core.model.color.RGBColorContainer import org.vitrivr.engine.core.model.content.element.ContentElement import org.vitrivr.engine.core.model.content.element.ImageContent import org.vitrivr.engine.core.model.retrievable.Ingested @@ -52,23 +51,29 @@ class RepresentativeImageContentAggregator : TransformerFactory { val firstImage = images.first() val height = firstImage.height val width = firstImage.width - val colors = List(firstImage.width * firstImage.height) { MutableRGBFloatColorContainer() } + val colors = List(firstImage.width * firstImage.height) { floatArrayOf(0.0f, 0.0f, 0.0f) } images.forEach { imageContent -> require(imageContent.height == height && imageContent.width == width) { "Unable to aggregate images! All images must have same dimension." } imageContent.content.getRGBArray().forEachIndexed { index, color -> - colors[index] += RGBByteColorContainer(color) + val rgb = RGBColorContainer(color) + colors[index][0] += rgb.red + colors[index][1] += rgb.green + colors[index][2] += rgb.blue } } /* normalize */ val div = images.size.toFloat() - colors.forEach { it /= div } + for (color in colors) { + color[0] /= div + color[1] /= div + color[2] /= div + } /* find image with smallest pixel-wise distance */ val mostRepresentative = images.minBy { imageContent -> - imageContent.content.getRGBArray().mapIndexed { index, color -> - RGBByteColorContainer(color).toFloatContainer().distanceTo(colors[index]) + RGBColorContainer(color).distanceTo(RGBColorContainer(colors[index])) }.sum() } diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogram.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogram.kt index 6aed6dc87..27ff72eaa 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogram.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogram.kt @@ -4,9 +4,8 @@ import org.vitrivr.engine.core.context.IndexContext import org.vitrivr.engine.core.context.QueryContext import org.vitrivr.engine.core.features.dense.DenseRetriever import org.vitrivr.engine.core.math.correspondence.LinearCorrespondence -import org.vitrivr.engine.core.model.color.ColorConverter -import org.vitrivr.engine.core.model.color.hsv.HSVColorContainer -import org.vitrivr.engine.core.model.color.rgb.RGBByteColorContainer +import org.vitrivr.engine.core.model.color.HSVColorContainer +import org.vitrivr.engine.core.model.color.RGBColorContainer import org.vitrivr.engine.core.model.content.Content import org.vitrivr.engine.core.model.content.element.ContentElement import org.vitrivr.engine.core.model.content.element.ImageContent @@ -131,7 +130,7 @@ class HueHistogram : Analyser { val hist = FloatArray(VECTOR_SIZE) val colors = content.content.getRGB(0, 0, content.content.width, content.content.height, null, 0, content.content.width) for (color in colors) { - val container: HSVColorContainer = ColorConverter.rgbToHsv(RGBByteColorContainer(color)) + val container: HSVColorContainer = RGBColorContainer(color).toHSV() if (container.saturation > 0.2f && container.value > 0.3f) { val h: Float = container.hue * hist.size val idx = h.toInt() From 4e34aaf9ffab6975a2a7ff37b63beee7d3e07f4b Mon Sep 17 00:00:00 2001 From: Ralph Gasser Date: Wed, 21 Aug 2024 11:30:00 +0200 Subject: [PATCH 09/20] Adds MedianColor feature. Signed-off-by: Ralph Gasser --- .../feature/mediancolor/MedianColor.kt | 143 ++++++++++++++++++ .../mediancolor/MedianColorExtractor.kt | 45 ++++++ ...trivr.engine.core.model.metamodel.Analyser | 2 + 3 files changed, 190 insertions(+) create mode 100644 vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/mediancolor/MedianColor.kt create mode 100644 vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/mediancolor/MedianColorExtractor.kt diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/mediancolor/MedianColor.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/mediancolor/MedianColor.kt new file mode 100644 index 000000000..b28ab3be0 --- /dev/null +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/mediancolor/MedianColor.kt @@ -0,0 +1,143 @@ +package org.vitrivr.engine.module.features.feature.mediancolor + +import org.vitrivr.engine.core.context.IndexContext +import org.vitrivr.engine.core.context.QueryContext +import org.vitrivr.engine.core.features.dense.DenseRetriever +import org.vitrivr.engine.core.math.correspondence.LinearCorrespondence +import org.vitrivr.engine.core.model.color.RGBColorContainer +import org.vitrivr.engine.core.model.content.Content +import org.vitrivr.engine.core.model.content.element.ImageContent +import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor +import org.vitrivr.engine.core.model.metamodel.Analyser +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.query.Query +import org.vitrivr.engine.core.model.query.proximity.ProximityQuery +import org.vitrivr.engine.core.model.retrievable.Retrievable +import org.vitrivr.engine.core.model.types.Value +import org.vitrivr.engine.core.operators.Operator +import org.vitrivr.engine.core.operators.ingest.Extractor +import org.vitrivr.engine.core.operators.retrieve.Retriever +import org.vitrivr.engine.core.util.math.StatisticsHelper.medianFromHistogram +import java.util.* + +/** + * Implementation of the [MedianColor] [Analyser], which derives the median color from an [ImageContent] as [FloatVectorDescriptor]. + * + * This [Analyser] has little practical relevance these days but acts as a simple example for how to create a custom [Analyser] that uses vectors. + * Furthermore, it can be used to implement test cases. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class MedianColor : Analyser { + + companion object { + private const val VECTOR_SIZE = 3 + } + + override val contentClasses = setOf(ImageContent::class) + override val descriptorClass = FloatVectorDescriptor::class + + /** + * Generates a prototypical [FloatVectorDescriptor] for this [MedianColor]. + * + * @param field [Schema.Field] to create the prototype for. + * @return [FloatVectorDescriptor] + */ + override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.FloatVector(VECTOR_SIZE)) + + /** + * Generates and returns a new [MedianColorExtractor] instance for this [MedianColor]. + * + * @param field The [Schema.Field] to create an [Extractor] for. + * @param input The [Operator] that acts as input to the new [Extractor]. + * @param context The [IndexContext] to use with the [Extractor]. + * + * @return A new [Extractor] instance for this [Analyser] + * @throws [UnsupportedOperationException], if this [Analyser] does not support the creation of an [Extractor] instance. + */ + override fun newExtractor(field: Schema.Field, input: Operator, context: IndexContext) = MedianColorExtractor(input, this, field) + + /** + * Generates and returns a new [MedianColorExtractor] instance for this [MedianColor]. + * + * @param name The name of the [MedianColorExtractor]. + * @param input The [Operator] that acts as input to the new [Extractor]. + * @param context The [IndexContext] to use with the [Extractor]. + * + * @return A new [Extractor] instance for this [Analyser] + * @throws [UnsupportedOperationException], if this [Analyser] does not support the creation of an [Extractor] instance. + */ + override fun newExtractor(name: String, input: Operator, context: IndexContext): Extractor = MedianColorExtractor(input, this, null) + + /** + * Generates and returns a new [DenseRetriever] instance for this [MedianColor]. + * + * @param field The [Schema.Field] to create an [Retriever] for. + * @param query The [Query] to use with the [Retriever]. + * @param context The [QueryContext] to use with the [Retriever]. + * + * @return A new [Retriever] instance for this [Analyser] + */ + override fun newRetrieverForQuery(field: Schema.Field, query: Query, context: QueryContext): DenseRetriever { + require(query is ProximityQuery<*> && query.value is Value.FloatVector) { "The query is not a ProximityQuery." } + @Suppress("UNCHECKED_CAST") + return DenseRetriever(field, query as ProximityQuery, context, LinearCorrespondence(3f)) + } + + /** + * Generates and returns a new [DenseRetriever] instance for this [MedianColor]. + * + * Invoking this method involves converting the provided [FloatVectorDescriptor] into a [ProximityQuery] that can be used to retrieve similar [ImageContent] elements. + * + * @param field The [Schema.Field] to create an [Retriever] for. + * @param descriptors An array of [FloatVectorDescriptor] elements to use with the [Retriever] + * @param context The [QueryContext] to use with the [Retriever] + */ + override fun newRetrieverForDescriptors(field: Schema.Field, descriptors: Collection, context: QueryContext): DenseRetriever { + /* Prepare query parameters. */ + val k = context.getProperty(field.fieldName, "limit")?.toLongOrNull() ?: 1000L + val fetchVector = context.getProperty(field.fieldName, "returnDescriptor")?.toBooleanStrictOrNull() ?: false + + /* Return retriever. */ + return this.newRetrieverForQuery(field, ProximityQuery(value = descriptors.first().vector, k = k, fetchVector = fetchVector), context) + } + + /** + * Generates and returns a new [DenseRetriever] instance for this [MedianColor]. + * + * Invoking this method involves converting the provided [ImageContent] and the [QueryContext] into a [FloatVectorDescriptor] + * that can be used to retrieve similar [ImageContent] elements. + * + * @param field The [Schema.Field] to create an [Retriever] for. + * @param content An array of [Content] elements to use with the [Retriever] + * @param context The [QueryContext] to use with the [Retriever] + */ + override fun newRetrieverForContent(field: Schema.Field, content: Collection, context: QueryContext): DenseRetriever = + this.newRetrieverForDescriptors(field, content.map { this.analyse(it) }, context) + + /** + * Performs the [MedianColor] analysis on the provided [List] of [ImageContent] elements. + * + * @param content The [List] of [ImageContent] elements. + * @return [FloatVectorDescriptor]s. + */ + fun analyse(content: ImageContent): FloatVectorDescriptor { + val r = IntArray(256) + val g = IntArray(256) + val b = IntArray(256) + + /* Extract colors from content and generate histogram. */ + val colors: IntArray = content.content.getRGB(0, 0, content.content.width, content.content.height, null, 0, content.content.width); + for (color in colors) { + val rgb = RGBColorContainer(color) + r[(rgb.red * 255).toInt()]++ + g[(rgb.green * 255).toInt()]++ + b[(rgb.blue * 255).toInt()]++ + } + + /* Generate vector from per-color histgrams. */ + val lab = RGBColorContainer(medianFromHistogram(r), medianFromHistogram(g), medianFromHistogram(b)).toXYZ().toLab() + return FloatVectorDescriptor(vector = lab.toVector()) + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/mediancolor/MedianColorExtractor.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/mediancolor/MedianColorExtractor.kt new file mode 100644 index 000000000..1773db968 --- /dev/null +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/mediancolor/MedianColorExtractor.kt @@ -0,0 +1,45 @@ +package org.vitrivr.engine.module.features.feature.mediancolor + +import org.vitrivr.engine.core.features.AbstractExtractor +import org.vitrivr.engine.core.features.metadata.source.file.FileSourceMetadataExtractor +import org.vitrivr.engine.core.model.content.ContentType +import org.vitrivr.engine.core.model.content.element.ImageContent +import org.vitrivr.engine.core.model.descriptor.Descriptor +import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.retrievable.Retrievable +import org.vitrivr.engine.core.operators.Operator +import org.vitrivr.engine.core.operators.ingest.Extractor +import org.vitrivr.engine.core.source.file.FileSource + + +/** + * [Extractor] implementation for the [MedianColor] analyser. + * + * @see [MedianColor] + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class MedianColorExtractor(input: Operator, analyser: MedianColor, field: Schema.Field?) : AbstractExtractor(input, analyser, field) { + /** + * Internal method to check, if [Retrievable] matches this [Extractor] and should thus be processed. + * + * [FileSourceMetadataExtractor] implementation only works with [Retrievable] that contain a [FileSource]. + * + * @param retrievable The [Retrievable] to check. + * @return True on match, false otherwise, + */ + override fun matches(retrievable: Retrievable): Boolean = retrievable.content.any { it.type == ContentType.BITMAP_IMAGE } + + /** + * Internal method to perform extraction on [Retrievable]. + ** + * @param retrievable The [Retrievable] to process. + * @return List of resulting [Descriptor]s. + */ + override fun extract(retrievable: Retrievable): List { + val content = retrievable.content.filterIsInstance() + return content.map { (this.analyser as MedianColor).analyse(it).copy(retrievableId = retrievable.id, field = this.field) } + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser b/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser index 032bde4d2..c11f16f14 100644 --- a/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser +++ b/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser @@ -1,4 +1,6 @@ org.vitrivr.engine.module.features.feature.ehd.EHD org.vitrivr.engine.module.features.feature.huehistogram.HueHistogram +org.vitrivr.engine.module.features.feature.mediancolor.MedianColor + org.vitrivr.engine.module.features.feature.external.implementations.dino.DINO org.vitrivr.engine.module.features.feature.external.implementations.clip.CLIP \ No newline at end of file From ae57e34ffd3c2147bf7f4d560f687cad0efd02ea Mon Sep 17 00:00:00 2001 From: Ralph Gasser Date: Wed, 21 Aug 2024 11:32:29 +0200 Subject: [PATCH 10/20] Adjusts AverageColor implementation so that it is in line with the other default features. Signed-off-by: Ralph Gasser --- .../module/features/feature/averagecolor/AverageColor.kt | 8 ++++---- .../feature/averagecolor/AverageColorExtractor.kt | 2 +- .../org.vitrivr.engine.core.model.metamodel.Analyser | 7 +++---- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/averagecolor/AverageColor.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/averagecolor/AverageColor.kt index 92d069d05..f1706b8d8 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/averagecolor/AverageColor.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/averagecolor/AverageColor.kt @@ -109,7 +109,7 @@ class AverageColor : Analyser { * @param context The [QueryContext] to use with the [Retriever] */ override fun newRetrieverForContent(field: Schema.Field, content: Collection, context: QueryContext): DenseRetriever = - this.newRetrieverForDescriptors(field, this.analyse(content), context) + this.newRetrieverForDescriptors(field, content.map { this.analyse(it) }, context) /** * Performs the [AverageColor] analysis on the provided [List] of [ImageContent] elements. @@ -117,9 +117,9 @@ class AverageColor : Analyser { * @param content The [List] of [ImageContent] elements. * @return [List] of [FloatVectorDescriptor]s. */ - fun analyse(content: Collection): List = content.map { + fun analyse(content: ImageContent): FloatVectorDescriptor { val color = floatArrayOf(0f, 0f, 0f) - val rgb = it.content.getRGBArray() + val rgb = content.content.getRGBArray() for (c in rgb) { val container = RGBColorContainer(c) color[0] += container.red @@ -129,6 +129,6 @@ class AverageColor : Analyser { /* Generate descriptor. */ val averageColor = RGBColorContainer(color[0] / rgb.size, color[1] / rgb.size, color[2] / rgb.size) - FloatVectorDescriptor(UUID.randomUUID(), null, averageColor.toVector()) + return FloatVectorDescriptor(UUID.randomUUID(), null, averageColor.toVector()) } } diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/averagecolor/AverageColorExtractor.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/averagecolor/AverageColorExtractor.kt index 6392bec1f..7a51f6716 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/averagecolor/AverageColorExtractor.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/averagecolor/AverageColorExtractor.kt @@ -39,6 +39,6 @@ class AverageColorExtractor(input: Operator, analyser: AverageColor */ override fun extract(retrievable: Retrievable): List { val content = retrievable.content.filterIsInstance() - return (this.analyser as AverageColor).analyse(content).map { it.copy(retrievableId = retrievable.id, field = this@AverageColorExtractor.field) } + return content.map { (this.analyser as AverageColor).analyse(it).copy(retrievableId = retrievable.id, field = this.field) } } } \ No newline at end of file diff --git a/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser b/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser index 9061f79da..296c11d6d 100644 --- a/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser +++ b/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser @@ -1,8 +1,7 @@ +org.vitrivr.engine.module.features.feature.averagecolor.AverageColor +org.vitrivr.engine.module.features.feature.dominantcolor.DominantColor org.vitrivr.engine.module.features.feature.ehd.EHD org.vitrivr.engine.module.features.feature.huehistogram.HueHistogram org.vitrivr.engine.module.features.feature.mediancolor.MedianColor - org.vitrivr.engine.module.features.feature.external.implementations.dino.DINO -org.vitrivr.engine.module.features.feature.external.implementations.clip.CLIP -org.vitrivr.engine.module.features.feature.averagecolor.AverageColor -org.vitrivr.engine.module.features.feature.dominantcolor.DominantColor +org.vitrivr.engine.module.features.feature.external.implementations.clip.CLIP \ No newline at end of file From 5b0e91264366c06914107ea0385bf53e87169dd8 Mon Sep 17 00:00:00 2001 From: Ralph Gasser Date: Wed, 21 Aug 2024 11:34:26 +0200 Subject: [PATCH 11/20] Fixes compilation error. Signed-off-by: Ralph Gasser --- .../feature/dominantcolor/DominantColor.kt | 59 ++++++++----------- 1 file changed, 23 insertions(+), 36 deletions(-) diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/dominantcolor/DominantColor.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/dominantcolor/DominantColor.kt index f44c86320..f102b92ff 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/dominantcolor/DominantColor.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/dominantcolor/DominantColor.kt @@ -2,9 +2,7 @@ package org.vitrivr.engine.module.features.feature.dominantcolor import org.vitrivr.engine.core.context.IndexContext import org.vitrivr.engine.core.context.QueryContext -import org.vitrivr.engine.module.features.feature.averagecolor.AverageColorExtractor -import org.vitrivr.engine.core.model.color.ColorConverter -import org.vitrivr.engine.core.model.color.rgb.RGBByteColorContainer +import org.vitrivr.engine.core.model.color.RGBColorContainer import org.vitrivr.engine.core.model.content.element.ImageContent import org.vitrivr.engine.core.model.descriptor.struct.LabelDescriptor import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor @@ -13,7 +11,6 @@ import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.query.Query import org.vitrivr.engine.core.model.query.bool.BooleanQuery import org.vitrivr.engine.core.model.query.bool.SimpleBooleanQuery -import org.vitrivr.engine.core.model.query.proximity.ProximityQuery import org.vitrivr.engine.core.model.retrievable.Retrievable import org.vitrivr.engine.core.model.types.Value import org.vitrivr.engine.core.operators.Operator @@ -111,39 +108,31 @@ class DominantColor : Analyser { val hist = IntArray(10) { 0 } val rgb = it.content.getRGBArray() rgb.forEach { color -> - val hsv = ColorConverter.rgbToHsv(RGBByteColorContainer(color)) + val hsv = RGBColorContainer(color).toHSV() if(hsv.saturation < 0.02f){ - ++hist[0]; - return@forEach; + ++hist[0] + return@forEach } - if(hsv.saturation < 0.2f || hsv.value < 0.3f){ - ++hist[9]; - return@forEach; - } - else if(hsv.hue >= 0.07 && hsv.hue < 0.14){ //orange - ++hist[2]; - } - else if(hsv.hue >= 0.14 && hsv.hue < 0.17){ //yellow - ++hist[3]; - } - else if(hsv.hue >= 0.17 && hsv.hue < 0.44){ //green - ++hist[4]; - } - else if(hsv.hue >= 0.44 && hsv.hue < 0.56){ //cyan - ++hist[5]; - } - else if(hsv.hue >= 0.56 && hsv.hue < 0.73){ //blue - ++hist[6]; - } - else if(hsv.hue >= 0.73 && hsv.hue < 0.76){ //violet - ++hist[7]; - } - else if(hsv.hue >= 0.76 && hsv.hue < 0.92){ //magenta - ++hist[8]; - } - else{ - ++hist[1]; + if (hsv.saturation < 0.2f || hsv.value < 0.3f) { + ++hist[9] + return@forEach + } else if (hsv.hue >= 0.07 && hsv.hue < 0.14) { //orange + ++hist[2] + } else if (hsv.hue >= 0.14 && hsv.hue < 0.17) { //yellow + ++hist[3] + } else if (hsv.hue >= 0.17 && hsv.hue < 0.44) { //green + ++hist[4] + } else if (hsv.hue >= 0.44 && hsv.hue < 0.56) { //cyan + ++hist[5] + } else if (hsv.hue >= 0.56 && hsv.hue < 0.73) { //blue + ++hist[6] + } else if (hsv.hue >= 0.73 && hsv.hue < 0.76) { //violet + ++hist[7] + } else if (hsv.hue >= 0.76 && hsv.hue < 0.92) { //magenta + ++hist[8] + } else { + ++hist[1] } } @@ -152,7 +141,5 @@ class DominantColor : Analyser { val label = if (max < rgb.size / 2) ColorLabel.UNDETERMINED else ColorLabel.entries[hist.indexOf(max)] return@map LabelDescriptor(UUID.randomUUID(), UUID.randomUUID(), label.name) - } - } \ No newline at end of file From 412ed2818502544c53335d8788587b3a4e9c11d9 Mon Sep 17 00:00:00 2001 From: Ralph Gasser Date: Wed, 21 Aug 2024 12:08:49 +0200 Subject: [PATCH 12/20] Simplifies HSV to RGB conversion. Signed-off-by: Ralph Gasser --- .../core/model/color/HSVColorContainer.kt | 60 ++++--------------- 1 file changed, 12 insertions(+), 48 deletions(-) diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/HSVColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/HSVColorContainer.kt index 54e00e146..70360c821 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/HSVColorContainer.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/HSVColorContainer.kt @@ -40,59 +40,23 @@ value class HSVColorContainer constructor(private val colors: FloatArray) { * @return [RGBColorContainer] */ fun toRGB(): RGBColorContainer { - var r = 0 - var g = 0 - var b = 0 if (this.saturation < 0.00001f) { - b = (this.value * 255f).toInt() - g = b - r = g + return RGBColorContainer(this.value, this.value, this.value) } else { - val h: Double = (this.hue * 6.0) % 6.0 + val h: Float = ((this.hue * 6.0) % 6.0).toFloat() val i = floor(h).toInt() - val v1 = (this.value * (1.0 - this.saturation) * 255f).toInt() - val v2 = (this.value * (1.0 - this.saturation * (h - i)) * 255f).toInt() - val v3 = (this.value * (1.0 - this.saturation * (1 - (h - i))) * 255f).toInt() - - when (i) { - 0 -> { - r = (255f * this.value).toInt() - g = v3 - b = v1 - } - - 1 -> { - r = v2 - g = (255f * this.value).toInt() - b = v1 - } - - 2 -> { - r = v1 - g = (255f * this.value).toInt() - b = v3 - } - - 3 -> { - r = v1 - g = v2 - b = (this.value * 255f).toInt() - } - - 4 -> { - r = v3 - g = v1 - b = (this.value * 255f).toInt() - } - - else -> { - r = (255f * this.value).toInt() - g = v1 - b = v2 - } + val v1 = (this.value * (1.0f - this.saturation)) + val v2 = (this.value * (1.0f - this.saturation * (h - i))) + val v3 = (this.value * (1.0f - this.saturation * (1.0f - (h - i)))) + return when (i) { + 0 -> RGBColorContainer(this.value, v3, v1) + 1 -> RGBColorContainer(v2, this.value, v1) + 2 -> RGBColorContainer(v1, this.value, v3) + 3 -> RGBColorContainer(v1, v2, 1.0f) + 4 -> RGBColorContainer(v3, v1, this.value) + else -> RGBColorContainer(this.value, v1, v2) } } - return RGBColorContainer(r, g, b) } override fun toString(): String = "HSVFloatColorContainer(H=$hue, S=$saturation, V=$value)" From 7e5a141e9124dd15b0365617e4138fd746e50a96 Mon Sep 17 00:00:00 2001 From: Ralph Gasser Date: Wed, 21 Aug 2024 12:20:25 +0200 Subject: [PATCH 13/20] Adds CLD feature. Signed-off-by: Ralph Gasser --- .../engine/core/model/color/ColorUtilities.kt | 35 +++ .../core/model/color/RGBColorContainer.kt | 23 +- .../core/model/color/YCbCrColorContainer.kt | 28 +++ .../engine/core/util/math/MathHelper.kt | 6 + .../engine/module/features/feature/cld/CLD.kt | 232 ++++++++++++++++++ .../features/feature/cld/CLDExtractor.kt | 44 ++++ .../engine/module/features/feature/ehd/EHD.kt | 5 +- ...trivr.engine.core.model.metamodel.Analyser | 1 + 8 files changed, 365 insertions(+), 9 deletions(-) create mode 100644 vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/ColorUtilities.kt create mode 100644 vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/YCbCrColorContainer.kt create mode 100644 vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/cld/CLD.kt create mode 100644 vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/cld/CLDExtractor.kt diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/ColorUtilities.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/ColorUtilities.kt new file mode 100644 index 000000000..a9883b506 --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/ColorUtilities.kt @@ -0,0 +1,35 @@ +package org.vitrivr.engine.core.model.color + +/** + * Utility class for color calculations. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +object ColorUtilities { + /** + * Calculates the average color of a collection of [RGBColorContainer]s. + * + * @param colors The [Collection] of [RGBColorContainer]s for which the average should be calculated. + * @return Average [RGBColorContainer]. + */ + fun avg(colors: Collection): RGBColorContainer { + var r = 0.0f + var g = 0.0f + var b = 0.0f + var alpha = 0.0f + var len = 0.0f + for (color in colors) { + alpha = color.alpha + r += color.red * color.alpha + g += color.green * color.alpha + b += color.blue * color.alpha + len += color.alpha + } + return if (len < 1f) { + RGBColorContainer(1.0f, 1.0f, 1.0f) + } else { + RGBColorContainer(r / len, g / len, b / len, alpha / len) + } + } +} \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBColorContainer.kt index a6b767b57..025873371 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBColorContainer.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBColorContainer.kt @@ -16,17 +16,19 @@ import kotlin.math.sqrt value class RGBColorContainer constructor(private val rgb: FloatArray) { init { - require(this.rgb.size == 3) { "RGBFloatColorContainer must have exactly 3 elements." } + require(this.rgb.size == 4) { "RGBFloatColorContainer must have exactly 3 elements." } require(this.rgb[0] in 0.0f..1.0f) { "RGBFloatColorContainer components must be between 0.0 and 1.0." } require(this.rgb[1] in 0.0f..1.0f) { "RGBFloatColorContainer components must be between 0.0 and 1.0." } require(this.rgb[2] in 0.0f..1.0f) { "RGBFloatColorContainer components must be between 0.0 and 1.0." } + require(this.rgb[3] in 0.0f..1.0f) { "RGBFloatColorContainer components must be between 0.0 and 1.0." } + } constructor(rgb: Int) : this((rgb shr 16 and 0xFF).toByte(), (rgb shr 8 and 0xFF).toByte(), (rgb and 0xFF).toByte()) - constructor(red: Byte, green: Byte, blue: Byte) : this(red.toFloat() / 255f, green.toFloat() / 255f, blue.toFloat() / 255f) - constructor(red: Int, green: Int, blue: Int) : this(red.toFloat() / 255f, green.toFloat() / 255f, blue.toFloat() / 255f) - constructor(r: Double, g: Double, b: Double) : this(r.toFloat(), g.toFloat(), b.toFloat()) - constructor(red: Float, green: Float, blue: Float) : this(floatArrayOf(red, green, blue)) + constructor(red: Byte, green: Byte, blue: Byte, alpha: Byte = Byte.MAX_VALUE) : this(red.toFloat() / 255f, green.toFloat() / 255f, blue.toFloat() / 255f, alpha.toFloat() / 255f) + constructor(red: Int, green: Int, blue: Int, alpha: Int = 255) : this(red.toFloat() / 255f, green.toFloat() / 255f, blue.toFloat() / 255f, alpha.toFloat() / 255f) + constructor(r: Double, g: Double, b: Double, alpha: Double = 1.0) : this(r.toFloat(), g.toFloat(), b.toFloat(), alpha.toFloat()) + constructor(red: Float, green: Float, blue: Float, alpha: Float = 1.0f) : this(floatArrayOf(red, green, blue, alpha)) /** Accessor for the red component of the [RGBColorContainer]. */ val red: Float @@ -40,6 +42,10 @@ value class RGBColorContainer constructor(private val rgb: FloatArray) { val blue: Float get() = this.rgb[2] + /** Accessor for the alpha component of the [RGBColorContainer]. */ + val alpha: Float + get() = this.rgb[3] + /** * Adds this [RGBColorContainer] to another [RGBColorContainer]. * @@ -142,6 +148,13 @@ value class RGBColorContainer constructor(private val rgb: FloatArray) { ) } + fun toYCbCr(): YCbCrColorContainer { + val y = Math.round((this.red * 255.0f * 65.738f + this.green * 255.0f * 129.057f + this.blue * 255.0f * 25.064f) / 256f + 16) + val cb = Math.round((this.red * 255.0f * -37.945f + this.green * 255.0f * -74.494f + this.blue * 255.0f * 112.439f) / 256f + 128) + val cr = Math.round((this.red * 255.0f * 112.439f + this.green * 255.0f * -94.154f + this.blue * 255.0f * -18.285f) / 256f + 128) + return YCbCrColorContainer(y, cb, cr) + } + /** * Converts this [RGBColorContainer] as a [List] of [Float]s. * diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/YCbCrColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/YCbCrColorContainer.kt new file mode 100644 index 000000000..eb8de6c2f --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/YCbCrColorContainer.kt @@ -0,0 +1,28 @@ +package org.vitrivr.engine.core.model.color + +/** + * A container for YCbCr colors. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +@JvmInline +value class YCbCrColorContainer constructor(private val ycbcr: IntArray) { + init { + require(this.ycbcr.size == 3) { "YCbCrColorContainer must have exactly 3 elements." } + } + + constructor(y: Int, cb: Int, cr: Int) : this(intArrayOf(y, cb, cr)) + + /** Accessor for the L component of the [LabColorContainer]. */ + val y: Int + get() = this.ycbcr[0] + + /** Accessor for the A component of the [LabColorContainer]. */ + val cb: Int + get() = this.ycbcr[1] + + /** Accessor for the B component of the [LabColorContainer]. */ + val cr: Int + get() = this.ycbcr[2] +} \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/util/math/MathHelper.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/util/math/MathHelper.kt index 07b1c5bea..c2bd52afd 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/util/math/MathHelper.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/util/math/MathHelper.kt @@ -18,6 +18,12 @@ object MathHelper { /** Square root of 2 as a [Float]. */ val SQRT2_f: Float = SQRT2.toFloat() + /** Square root of 1/2. */ + val SQRT1_2: Double = sqrt(0.5) + + /** Square root of 1/2 as a [Float]. */ + val SQRT1_2f: Float = SQRT1_2.toFloat() + /** * Normalizes a [Array] of [Value.Float] with respect to the L2 (euclidian) norm. * The method will return a new array and leave the original array unchanged. diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/cld/CLD.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/cld/CLD.kt new file mode 100644 index 000000000..f3b126877 --- /dev/null +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/cld/CLD.kt @@ -0,0 +1,232 @@ +package org.vitrivr.engine.module.features.feature.cld + +import org.vitrivr.engine.core.context.IndexContext +import org.vitrivr.engine.core.context.QueryContext +import org.vitrivr.engine.core.features.dense.DenseRetriever +import org.vitrivr.engine.core.math.correspondence.LinearCorrespondence +import org.vitrivr.engine.core.model.color.ColorUtilities +import org.vitrivr.engine.core.model.color.RGBColorContainer +import org.vitrivr.engine.core.model.content.Content +import org.vitrivr.engine.core.model.content.element.ContentElement +import org.vitrivr.engine.core.model.content.element.ImageContent +import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor +import org.vitrivr.engine.core.model.metamodel.Analyser +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.query.Query +import org.vitrivr.engine.core.model.query.proximity.ProximityQuery +import org.vitrivr.engine.core.model.retrievable.Retrievable +import org.vitrivr.engine.core.model.types.Value +import org.vitrivr.engine.core.operators.Operator +import org.vitrivr.engine.core.operators.ingest.Extractor +import org.vitrivr.engine.core.operators.retrieve.Retriever +import org.vitrivr.engine.core.util.extension.getRGBArray +import org.vitrivr.engine.core.util.math.MathHelper.SQRT1_2 +import java.util.* +import kotlin.math.cos +import kotlin.math.floor +import kotlin.reflect.KClass + +/** + * An MPEG 7 Color Layout (CLD) [Analyser] for [ImageContent] objects. + * + * Migrated from Cineast. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class CLD : Analyser { + companion object { + private const val VECTOR_SIZE = 12 + + + private val SCAN: IntArray = intArrayOf( + 0, 1, 8, 16, 9, 2, 3, 10, 17, 24, 32, + 25, 18, 11, 4, 5, 12, 19, 26, 33, 40, 48, 41, 34, 27, 20, 13, 6, 7, + 14, 21, 28, 35, 42, 49, 56, 57, 50, 43, 36, 29, 22, 15, 23, 30, 37, + 44, 51, 58, 59, 52, 45, 38, 31, 39, 46, 53, 60, 61, 54, 47, 55, 62, + 63 + ) + } + + override val contentClasses: Set>> = setOf(ImageContent::class) + override val descriptorClass: KClass = FloatVectorDescriptor::class + + /** + * Generates a prototypical [FloatVectorDescriptor] for this [CLD]. + * + * @param field [Schema.Field] to create the prototype for. + * @return [FloatVectorDescriptor] + */ + override fun prototype(field: Schema.Field<*, *>): FloatVectorDescriptor = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.FloatVector(VECTOR_SIZE)) + + /** + * Generates and returns a new [CLDExtractor] instance for this [CLD]. + * + * @param name The name of the [CLDExtractor]. + * @param input The [Operator] that acts as input to the new [Extractor]. + * @param context The [IndexContext] to use with the [Extractor]. + * + * @return A new [Extractor] instance for this [Analyser] + * @throws [UnsupportedOperationException], if this [Analyser] does not support the creation of an [Extractor] instance. + */ + override fun newExtractor(name: String, input: Operator, context: IndexContext) = CLDExtractor(input, this, null) + + /** + * Generates and returns a new [CLDExtractor] instance for this [CLD]. + * + * @param field The [Schema.Field] to create an [Extractor] for. + * @param input The [Operator] that acts as input to the new [Extractor]. + * @param context The [IndexContext] to use with the [Extractor]. + * + * @return A new [Extractor] instance for this [Analyser] + * @throws [UnsupportedOperationException], if this [Analyser] does not support the creation of an [Extractor] instance. + */ + override fun newExtractor(field: Schema.Field, input: Operator, context: IndexContext) = CLDExtractor(input, this, field) + + /** + * Generates and returns a new [DenseRetriever] instance for this [CLD]. + * + * @param field The [Schema.Field] to create an [Retriever] for. + * @param query The [Query] to use with the [Retriever]. + * @param context The [QueryContext] to use with the [Retriever]. + * + * @return A new [DenseRetriever] instance for this [CLD] + */ + override fun newRetrieverForQuery(field: Schema.Field, query: Query, context: QueryContext): DenseRetriever { + require(query is ProximityQuery<*> && query.value is Value.FloatVector) { "The query is not a ProximityQuery." } + @Suppress("UNCHECKED_CAST") + return DenseRetriever(field, query as ProximityQuery, context, LinearCorrespondence(490f)) + } + + /** + * Generates and returns a new [DenseRetriever] instance for this [CLD]. + * + * Invoking this method involves converting the provided [FloatVectorDescriptor] into a [ProximityQuery] that can be used to retrieve similar [ImageContent] elements. + * + * @param field The [Schema.Field] to create an [Retriever] for. + * @param descriptors An array of [FloatVectorDescriptor] elements to use with the [Retriever] + * @param context The [QueryContext] to use with the [Retriever] + */ + override fun newRetrieverForDescriptors(field: Schema.Field, descriptors: Collection, context: QueryContext): DenseRetriever { + /* Prepare query parameters. */ + val k = context.getProperty(field.fieldName, "limit")?.toLongOrNull() ?: 1000L + val fetchVector = context.getProperty(field.fieldName, "returnDescriptor")?.toBooleanStrictOrNull() ?: false + + /* Return retriever. */ + return this.newRetrieverForQuery(field, ProximityQuery(value = descriptors.first().vector, k = k, fetchVector = fetchVector), context) + } + + /** + * Generates and returns a new [DenseRetriever] instance for this [CLD]. + * + * Invoking this method involves converting the provided [ImageContent] and the [QueryContext] into a [FloatVectorDescriptor] + * that can be used to retrieve similar [ImageContent] elements. + * + * @param field The [Schema.Field] to create an [Retriever] for. + * @param content An array of [Content] elements to use with the [Retriever] + * @param context The [QueryContext] to use with the [Retriever] + */ + override fun newRetrieverForContent(field: Schema.Field, content: Collection, context: QueryContext) = this.newRetrieverForDescriptors(field, content.map { this.analyse(it) }, context) + + /** + * Performs the [CLD] analysis on the provided [ImageContent] and returns a [FloatVectorDescriptor] that represents the result. + * + * @param content [ImageContent] to be analysed. + * @return [FloatVectorDescriptor] result of the analysis. + */ + fun analyse(content: ImageContent): FloatVectorDescriptor { + val tmpList: ArrayList = ArrayList(content.content.width * content.content.height) + + /* Extract and normalize colors based on alpha channel. */ + val colors: IntArray = content.content.getRGBArray() + for (c in colors) { + val rgb = RGBColorContainer(c) + if (rgb.alpha < 127) { + tmpList.add(255) + } else { + tmpList.add(c) + } + } + + /* Create partitions. */ + val partitions: ArrayList> = partition(tmpList, content.content.width, content.content.height) + val rgbs = IntArray(64) + for (i in partitions.indices) { + rgbs[i] = ColorUtilities.avg(partitions[i].map { RGBColorContainer(it) }).toRGBInt() + } + + /* Obtain YCbCr values and calculate DCT. */ + val ycbcrs = Array(3) { IntArray(64) } + for (i in 0..63) { + val c = RGBColorContainer(rgbs[i]).toYCbCr() + ycbcrs[0][i] = c.y + ycbcrs[1][i] = c.cb + ycbcrs[2][i] = c.cr + } + ycbcrs[0] = this.dct(ycbcrs[0]) + ycbcrs[1] = this.dct(ycbcrs[1]) + ycbcrs[2] = this.dct(ycbcrs[2]) + + /* Obtain CLD. */ + val cld = Value.FloatVector( + floatArrayOf( + ycbcrs[0][0].toFloat(), ycbcrs[0][1].toFloat(), ycbcrs[0][2].toFloat(), ycbcrs[0][3].toFloat(), ycbcrs[0][4].toFloat(), ycbcrs[0][5].toFloat(), + ycbcrs[1][0].toFloat(), ycbcrs[1][1].toFloat(), ycbcrs[1][2].toFloat(), + ycbcrs[2][0].toFloat(), ycbcrs[2][1].toFloat(), ycbcrs[2][2].toFloat() + ) + ) + return FloatVectorDescriptor(vector = cld) + } + + /** + * Calculates the Discrete Cosine Transform (DCT) of a given block. + * + * Based on c implementation by Berk ATABEK (http://www.batabek.com/) + * + * @param block Block for which to calculate the DCT. + * @return The DCT of the block. + */ + private fun dct(block: IntArray): IntArray { + var sum: Double + var cu: Double + var cv: Double + val temp = IntArray(64) + + for (u in 0..7) { + for (v in 0..7) { + sum = 0.0 + cu = if ((u == 0)) SQRT1_2 else 1.0 + cv = if ((v == 0)) SQRT1_2 else 1.0 + for (x in 0..7) { + for (y in 0..7) { + sum += (block[x * 8 + y] * cos((2 * x + 1) * u * Math.PI / 16.0) * cos((2 * y + 1) * v * Math.PI / 16.0)) + } + } + temp[SCAN[8 * u + v]] = floor((0.25 * cu * cv * sum) + 0.5).toInt() + } + } + return temp + } + + /** + * Generates 8 x 8 partitions of the input list. + * + * @param input List to partition. + * @param width Width of the image. + * @param height Height of the image. + * @return List of 8 x 8 partitions. + */ + private fun partition(input: List, width: Int, height: Int): ArrayList> { + val ret = ArrayList>(8 * 8) + for (i in 0 until 8 * 8) { + ret.add(LinkedList()) + } + + for ((i, t) in input.withIndex()) { + val index = (((i % width) * 8) / width) + 8 * (i * 8 / width / height) + ret[index].add(t) + } + + return ret + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/cld/CLDExtractor.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/cld/CLDExtractor.kt new file mode 100644 index 000000000..68e128570 --- /dev/null +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/cld/CLDExtractor.kt @@ -0,0 +1,44 @@ +package org.vitrivr.engine.module.features.feature.cld + +import org.vitrivr.engine.core.features.AbstractExtractor +import org.vitrivr.engine.core.features.metadata.source.file.FileSourceMetadataExtractor +import org.vitrivr.engine.core.model.content.ContentType +import org.vitrivr.engine.core.model.content.element.ImageContent +import org.vitrivr.engine.core.model.descriptor.Descriptor +import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.retrievable.Retrievable +import org.vitrivr.engine.core.operators.Operator +import org.vitrivr.engine.core.operators.ingest.Extractor +import org.vitrivr.engine.core.source.file.FileSource + +/** + * [Extractor] implementation for the [CLD] analyser. + * + * @see [CLD] + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class CLDExtractor(input: Operator, analyser: CLD, field: Schema.Field?) : AbstractExtractor(input, analyser, field) { + /** + * Internal method to check, if [Retrievable] matches this [Extractor] and should thus be processed. + * + * [FileSourceMetadataExtractor] implementation only works with [Retrievable] that contain a [FileSource]. + * + * @param retrievable The [Retrievable] to check. + * @return True on match, false otherwise, + */ + override fun matches(retrievable: Retrievable): Boolean = retrievable.content.any { it.type == ContentType.BITMAP_IMAGE } + + /** + * Internal method to perform extraction on [Retrievable]. + * + * @param retrievable The [Retrievable] to process. + * @return List of resulting [Descriptor]s. + */ + override fun extract(retrievable: Retrievable): List { + val content = retrievable.content.filterIsInstance() + return content.map { (this.analyser as CLD).analyse(it).copy(retrievableId = retrievable.id, field = this.field) } + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/ehd/EHD.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/ehd/EHD.kt index 1c752cd9c..533770e77 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/ehd/EHD.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/ehd/EHD.kt @@ -24,7 +24,7 @@ import java.util.* import kotlin.reflect.KClass /** - * A MPEG 7 Edge Histogram Descriptor (EHD) [Analyser] for [ImageContent] objects. + * An MPEG 7 Edge Histogram Descriptor (EHD) [Analyser] for [ImageContent] objects. * * Migrated from Cineast. * @@ -113,7 +113,6 @@ class EHD : Analyser { * @return A new [DenseRetriever] instance for this [EHD] */ override fun newRetrieverForQuery(field: Schema.Field, query: Query, context: QueryContext): DenseRetriever { - require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } require(query is ProximityQuery<*> && query.value is Value.FloatVector) { "The query is not a ProximityQuery." } @Suppress("UNCHECKED_CAST") return DenseRetriever(field, query as ProximityQuery, context, LinearCorrespondence(4f)) @@ -129,8 +128,6 @@ class EHD : Analyser { * @param context The [QueryContext] to use with the [Retriever] */ override fun newRetrieverForDescriptors(field: Schema.Field, descriptors: Collection, context: QueryContext): DenseRetriever { - require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } - /* Prepare query parameters. */ val k = context.getProperty(field.fieldName, "limit")?.toLongOrNull() ?: 1000L val fetchVector = context.getProperty(field.fieldName, "returnDescriptor")?.toBooleanStrictOrNull() ?: false diff --git a/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser b/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser index 296c11d6d..83e584a79 100644 --- a/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser +++ b/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser @@ -1,4 +1,5 @@ org.vitrivr.engine.module.features.feature.averagecolor.AverageColor +org.vitrivr.engine.module.features.feature.cld.CLD org.vitrivr.engine.module.features.feature.dominantcolor.DominantColor org.vitrivr.engine.module.features.feature.ehd.EHD org.vitrivr.engine.module.features.feature.huehistogram.HueHistogram From 2a8f564d6a283f4ff5e1c51d25e439bdd0f3bdcd Mon Sep 17 00:00:00 2001 From: Ralph Gasser Date: Wed, 21 Aug 2024 12:24:12 +0200 Subject: [PATCH 14/20] Moves AverageColor back to core module, because some unit tests rely on it. Signed-off-by: Ralph Gasser --- .../vitrivr/engine/core/features}/averagecolor/AverageColor.kt | 2 +- .../engine/core/features}/averagecolor/AverageColorExtractor.kt | 2 +- .../services/org.vitrivr.engine.core.model.metamodel.Analyser | 1 + .../services/org.vitrivr.engine.core.model.metamodel.Analyser | 1 - 4 files changed, 3 insertions(+), 3 deletions(-) rename {vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature => vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features}/averagecolor/AverageColor.kt (99%) rename {vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature => vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features}/averagecolor/AverageColorExtractor.kt (96%) diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/averagecolor/AverageColor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColor.kt similarity index 99% rename from vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/averagecolor/AverageColor.kt rename to vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColor.kt index f1706b8d8..fc307eaf0 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/averagecolor/AverageColor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColor.kt @@ -1,4 +1,4 @@ -package org.vitrivr.engine.module.features.feature.averagecolor +package org.vitrivr.engine.core.features.averagecolor import org.vitrivr.engine.core.context.IndexContext import org.vitrivr.engine.core.context.QueryContext diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/averagecolor/AverageColorExtractor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColorExtractor.kt similarity index 96% rename from vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/averagecolor/AverageColorExtractor.kt rename to vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColorExtractor.kt index 7a51f6716..41ffcd315 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/averagecolor/AverageColorExtractor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColorExtractor.kt @@ -1,4 +1,4 @@ -package org.vitrivr.engine.module.features.feature.averagecolor +package org.vitrivr.engine.core.features.averagecolor import org.vitrivr.engine.core.features.AbstractExtractor import org.vitrivr.engine.core.features.metadata.source.file.FileSourceMetadataExtractor diff --git a/vitrivr-engine-core/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser b/vitrivr-engine-core/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser index e9d29c3eb..f3d140936 100644 --- a/vitrivr-engine-core/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser +++ b/vitrivr-engine-core/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser @@ -1,3 +1,4 @@ +org.vitrivr.engine.core.features.averagecolor.AverageColor org.vitrivr.engine.core.features.metadata.source.file.FileSourceMetadata org.vitrivr.engine.core.features.metadata.source.exif.ExifMetadata org.vitrivr.engine.core.features.metadata.source.video.VideoSourceMetadata diff --git a/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser b/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser index 83e584a79..2aea6b065 100644 --- a/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser +++ b/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser @@ -1,4 +1,3 @@ -org.vitrivr.engine.module.features.feature.averagecolor.AverageColor org.vitrivr.engine.module.features.feature.cld.CLD org.vitrivr.engine.module.features.feature.dominantcolor.DominantColor org.vitrivr.engine.module.features.feature.ehd.EHD From 3168ae3ff63b534c0ec691aecb47730b5bb2e17e Mon Sep 17 00:00:00 2001 From: Ralph Gasser Date: Wed, 21 Aug 2024 12:47:01 +0200 Subject: [PATCH 15/20] Color conversions now use the standard library. Signed-off-by: Ralph Gasser --- .../core/model/color/HSVColorContainer.kt | 39 ++++------ .../core/model/color/LabColorContainer.kt | 25 +++++-- .../core/model/color/RGBColorContainer.kt | 71 ++++++------------ .../core/model/color/XYZColorContainer.kt | 75 +++---------------- .../core/model/color/YCbCrColorContainer.kt | 23 ++++-- .../engine/module/features/feature/cld/CLD.kt | 8 +- .../feature/mediancolor/MedianColor.kt | 4 +- 7 files changed, 93 insertions(+), 152 deletions(-) diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/HSVColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/HSVColorContainer.kt index 70360c821..080735fd4 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/HSVColorContainer.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/HSVColorContainer.kt @@ -1,7 +1,7 @@ package org.vitrivr.engine.core.model.color import org.vitrivr.engine.core.model.types.Value -import kotlin.math.floor +import java.awt.color.ColorSpace /** * A container for HSV colors. @@ -10,29 +10,36 @@ import kotlin.math.floor * @version 1.0.0 */ @JvmInline -value class HSVColorContainer constructor(private val colors: FloatArray) { +value class HSVColorContainer constructor(private val hsv: FloatArray) { + + init { + require(this.hsv.size == 3) { "HSVFloatColorContainer must have exactly 3 elements." } + require(this.hsv[0] in 0.0f..1.0f) { "HSVFloatColorContainer components must be between 0.0 and 1.0." } + require(this.hsv[1] in 0.0f..1.0f) { "HSVFloatColorContainer components must be between 0.0 and 1.0." } + require(this.hsv[2] in 0.0f..1.0f) { "HSVFloatColorContainer components must be between 0.0 and 1.0." } + } constructor(hue: Float, saturation: Float, value: Float) : this(floatArrayOf(hue, saturation, value)) constructor(hue: Double, saturation: Double, value: Double) : this(floatArrayOf(hue.toFloat(), saturation.toFloat(), value.toFloat())) /** Accessor for the hue component of the [HSVColorContainer]. */ val hue: Float - get() = this.colors[0] + get() = this.hsv[0] /** Accessor for the saturation component of the [HSVColorContainer]. */ val saturation: Float - get() = this.colors[1] + get() = this.hsv[1] /** Accessor for the value component of the [HSVColorContainer]. */ val value: Float - get() = this.colors[2] + get() = this.hsv[2] /** * Converts this [HSVColorContainer] a [Value.FloatVector] * * @return [Value.FloatVector] */ - fun toVector() = Value.FloatVector(this.colors) + fun toVector() = Value.FloatVector(this.hsv) /** * Converts this [HSVColorContainer] to a [RGBColorContainer]. @@ -40,23 +47,9 @@ value class HSVColorContainer constructor(private val colors: FloatArray) { * @return [RGBColorContainer] */ fun toRGB(): RGBColorContainer { - if (this.saturation < 0.00001f) { - return RGBColorContainer(this.value, this.value, this.value) - } else { - val h: Float = ((this.hue * 6.0) % 6.0).toFloat() - val i = floor(h).toInt() - val v1 = (this.value * (1.0f - this.saturation)) - val v2 = (this.value * (1.0f - this.saturation * (h - i))) - val v3 = (this.value * (1.0f - this.saturation * (1.0f - (h - i)))) - return when (i) { - 0 -> RGBColorContainer(this.value, v3, v1) - 1 -> RGBColorContainer(v2, this.value, v1) - 2 -> RGBColorContainer(v1, this.value, v3) - 3 -> RGBColorContainer(v1, v2, 1.0f) - 4 -> RGBColorContainer(v3, v1, this.value) - else -> RGBColorContainer(this.value, v1, v2) - } - } + val space = ColorSpace.getInstance(ColorSpace.TYPE_HSV) + val hsv = space.toRGB(this.hsv) + return RGBColorContainer(hsv[0], hsv[1], hsv[2]) } override fun toString(): String = "HSVFloatColorContainer(H=$hue, S=$saturation, V=$value)" diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/LabColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/LabColorContainer.kt index 3e1c194a2..d8027b85c 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/LabColorContainer.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/LabColorContainer.kt @@ -1,6 +1,7 @@ package org.vitrivr.engine.core.model.color import org.vitrivr.engine.core.model.types.Value +import java.awt.color.ColorSpace /** * A container for LAB colors. @@ -9,29 +10,43 @@ import org.vitrivr.engine.core.model.types.Value * @version 1.0.0 */ @JvmInline -value class LabColorContainer constructor(private val colors: FloatArray) { +value class LabColorContainer constructor(private val lab: FloatArray) { + init { + require(this.lab.size == 3) { "LabColorContainer must have exactly 3 elements." } + } constructor(l: Float, a: Float, b: Float) : this(floatArrayOf(l, a, b)) constructor(l: Double, a: Double, b: Double) : this(floatArrayOf(l.toFloat(), a.toFloat(), b.toFloat())) /** Accessor for the L component of the [LabColorContainer]. */ val l: Float - get() = this.colors[0] + get() = this.lab[0] /** Accessor for the A component of the [LabColorContainer]. */ val a: Float - get() = this.colors[1] + get() = this.lab[1] /** Accessor for the B component of the [LabColorContainer]. */ val b: Float - get() = this.colors[2] + get() = this.lab[2] /** * Converts this [LabColorContainer] a [Value.FloatVector] * * @return [Value.FloatVector] */ - fun toVector() = Value.FloatVector(this.colors) + fun toVector() = Value.FloatVector(this.lab) + + /** + * Converts this [XYZColorContainer] to a [RGBColorContainer]. + * + * @return [RGBColorContainer] + */ + fun toRGB(): RGBColorContainer { + val space = ColorSpace.getInstance(ColorSpace.TYPE_Lab) + val lab = space.toRGB(this.lab) + return RGBColorContainer(lab[0], lab[1], lab[2]) + } override fun toString(): String = "LabFloatColorContainer(L=$l, A=$a, B=$b)" } \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBColorContainer.kt index 025873371..a1dac81c2 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBColorContainer.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBColorContainer.kt @@ -1,9 +1,9 @@ package org.vitrivr.engine.core.model.color import org.vitrivr.engine.core.model.types.Value +import java.awt.color.ColorSpace import kotlin.math.max import kotlin.math.min -import kotlin.math.pow import kotlin.math.sqrt /** @@ -16,12 +16,11 @@ import kotlin.math.sqrt value class RGBColorContainer constructor(private val rgb: FloatArray) { init { - require(this.rgb.size == 4) { "RGBFloatColorContainer must have exactly 3 elements." } + require(this.rgb.size == 4) { "RGBFloatColorContainer must have exactly 4 elements." } require(this.rgb[0] in 0.0f..1.0f) { "RGBFloatColorContainer components must be between 0.0 and 1.0." } require(this.rgb[1] in 0.0f..1.0f) { "RGBFloatColorContainer components must be between 0.0 and 1.0." } require(this.rgb[2] in 0.0f..1.0f) { "RGBFloatColorContainer components must be between 0.0 and 1.0." } require(this.rgb[3] in 0.0f..1.0f) { "RGBFloatColorContainer components must be between 0.0 and 1.0." } - } constructor(rgb: Int) : this((rgb shr 16 and 0xFF).toByte(), (rgb shr 8 and 0xFF).toByte(), (rgb and 0xFF).toByte()) @@ -97,17 +96,18 @@ value class RGBColorContainer constructor(private val rgb: FloatArray) { * @return [HSVColorContainer] */ fun toHSV(): HSVColorContainer { - val max = maxOf(this.red, this.green, this.blue) - val min = minOf(this.red, this.green, this.blue) - val d = max - min - val s = if (max == 0f) 0f else d / max - val h = when { - d == 0f -> 0f - max == this.red -> (this.green - this.blue) / d + (if (this.green < this.blue) 6 else 0) - max == this.green -> (this.blue - this.red) / d + 2 - else -> (this.red - this.green) / d + 4 - } - return HSVColorContainer(h, s, max) + val space = ColorSpace.getInstance(ColorSpace.TYPE_HSV) + return HSVColorContainer(space.fromRGB(this.rgb)) + } + + /** + * Converts this [RGBColorContainer] to a [LabColorContainer]. + * + * @return [LabColorContainer] + */ + fun toLab(): LabColorContainer { + val space = ColorSpace.getInstance(ColorSpace.TYPE_Lab) + return LabColorContainer(space.fromRGB(this.rgb)) } /** @@ -116,43 +116,18 @@ value class RGBColorContainer constructor(private val rgb: FloatArray) { * @return [XYZColorContainer] */ fun toXYZ(): XYZColorContainer { - var r = this.red.toDouble() - var g = this.green.toDouble() - var b = this.blue.toDouble() - if (r > 0.04045) { - r = ((r + 0.055) / 1.055).pow(2.4) - } else { - r /= 12.92 - } - - if (g > 0.04045) { - g = ((g + 0.055) / 1.055).pow(2.4) - } else { - g /= 12.92 - } - - if (b > 0.04045) { - b = ((b + 0.055) / 1.055).pow(2.4) - } else { - b /= 12.92 - } - - r *= 100.0 - g *= 100.0 - b *= 100.0 - - return XYZColorContainer( - r * 0.4121 + g * 0.3576 + b * 0.1805, - r * 0.2126 + g * 0.7152 + b * 0.0722, - r * 0.0193 + g * 0.1192 + b * 0.9505 - ) + val space = ColorSpace.getInstance(ColorSpace.TYPE_XYZ) + return XYZColorContainer(space.fromRGB(this.rgb)) } + /** + * Converts this [RGBColorContainer] to a [YCbCrColorContainer]. + * + * @return [YCbCrColorContainer] + */ fun toYCbCr(): YCbCrColorContainer { - val y = Math.round((this.red * 255.0f * 65.738f + this.green * 255.0f * 129.057f + this.blue * 255.0f * 25.064f) / 256f + 16) - val cb = Math.round((this.red * 255.0f * -37.945f + this.green * 255.0f * -74.494f + this.blue * 255.0f * 112.439f) / 256f + 128) - val cr = Math.round((this.red * 255.0f * 112.439f + this.green * 255.0f * -94.154f + this.blue * 255.0f * -18.285f) / 256f + 128) - return YCbCrColorContainer(y, cb, cr) + val space = ColorSpace.getInstance(ColorSpace.TYPE_YCbCr) + return YCbCrColorContainer(space.fromRGB(this.rgb)) } /** diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/XYZColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/XYZColorContainer.kt index 27591c1da..f6b5b0cd6 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/XYZColorContainer.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/XYZColorContainer.kt @@ -1,7 +1,7 @@ package org.vitrivr.engine.core.model.color import org.vitrivr.engine.core.model.types.Value -import kotlin.math.pow +import java.awt.color.ColorSpace /** * A container for XYZ colors. @@ -10,87 +10,32 @@ import kotlin.math.pow * @version 1.0.0 */ @JvmInline -value class XYZColorContainer constructor(private val colors: FloatArray) { +value class XYZColorContainer constructor(private val xyz: FloatArray) { constructor(x: Float, y: Float, z: Float) : this(floatArrayOf(x, y, z)) constructor(x: Double, y: Double, z: Double) : this(floatArrayOf(x.toFloat(), y.toFloat(), z.toFloat())) /** Accessor for the L component of the [LabColorContainer]. */ val x: Float - get() = this.colors[0] + get() = this.xyz[0] /** Accessor for the A component of the [LabColorContainer]. */ val y: Float - get() = this.colors[1] + get() = this.xyz[1] /** Accessor for the B component of the [LabColorContainer]. */ val z: Float - get() = this.colors[2] - - /** - * Converts this [XYZColorContainer] to a [LabColorContainer]. - * - * @return [LabColorContainer] - */ - fun toLab(): LabColorContainer { - var x = this.x / 95.047 - var y = this.y / 100.0 - var z = this.z / 108.883 - - if (x > 0.008856f) { - x = x.pow((1.0 / 3.0)) - } else { - x = (x * 7.787 + (16.0 / 116.0)) - } - - y = if (y > 0.008856f) { - y.pow((1.0 / 3.0)) - } else { - (y * 7.787 + (16.0 / 116.0)) - } - - z = if (z > 0.008856f) { - z.pow((1.0 / 3.0)) - } else { - (z * 7.787 + (16.0 / 116.0)) - } - - return LabColorContainer((116.0 * y) - 16.0, 500.0 * (x - y), 200.0 * (y - z)) - } + get() = this.xyz[2] /** * Converts this [XYZColorContainer] to a [RGBColorContainer]. * - * @return [LabColorContainer] + * @return [RGBColorContainer] */ fun toRGB(): RGBColorContainer { - val x: Double = this.x / 100.0 - val y: Double = this.y / 100.0 - val z: Double = this.z / 100.0 - - var r = x * 3.2406 + y * -1.5372 + z * -0.4986 - var g = x * -0.9689 + y * 1.8758 + z * 0.0415 - var b = x * 0.0557 + y * -0.2040 + z * 1.0570 - - if (r > 0.0031308) { - r = 1.055 * r.pow((1.0 / 2.4)) - 0.055 - } else { - r *= 12.92 - } - - if (g > 0.0031308) { - g = 1.055 * g.pow((1.0 / 2.4)) - 0.055 - } else { - g *= 12.92 - } - - if (b > 0.0031308) { - b = 1.055 * b.pow((1.0 / 2.4)) - 0.055 - } else { - b *= 12.92 - } - - return RGBColorContainer(r, g, b) + val space = ColorSpace.getInstance(ColorSpace.TYPE_XYZ) + val xyz = space.toRGB(this.xyz) + return RGBColorContainer(xyz[0], xyz[1], xyz[2]) } /** @@ -98,7 +43,7 @@ value class XYZColorContainer constructor(private val colors: FloatArray) { * * @return [Value.FloatVector] */ - fun toVector() = Value.FloatVector(this.colors) + fun toVector() = Value.FloatVector(this.xyz) override fun toString(): String { return "XYZFloatColorContainer(X=$x, Y=$y, Z=$z)" diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/YCbCrColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/YCbCrColorContainer.kt index eb8de6c2f..5eef673aa 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/YCbCrColorContainer.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/YCbCrColorContainer.kt @@ -1,5 +1,7 @@ package org.vitrivr.engine.core.model.color +import java.awt.color.ColorSpace + /** * A container for YCbCr colors. * @@ -7,22 +9,33 @@ package org.vitrivr.engine.core.model.color * @version 1.0.0 */ @JvmInline -value class YCbCrColorContainer constructor(private val ycbcr: IntArray) { +value class YCbCrColorContainer constructor(private val ycbcr: FloatArray) { init { require(this.ycbcr.size == 3) { "YCbCrColorContainer must have exactly 3 elements." } } - constructor(y: Int, cb: Int, cr: Int) : this(intArrayOf(y, cb, cr)) + constructor(y: Float, cb: Float, cr: Float) : this(floatArrayOf(y, cb, cr)) /** Accessor for the L component of the [LabColorContainer]. */ - val y: Int + val y: Float get() = this.ycbcr[0] /** Accessor for the A component of the [LabColorContainer]. */ - val cb: Int + val cb: Float get() = this.ycbcr[1] /** Accessor for the B component of the [LabColorContainer]. */ - val cr: Int + val cr: Float get() = this.ycbcr[2] + + /** + * Converts this [YCbCrColorContainer] to a [RGBColorContainer]. + * + * @return [RGBColorContainer] + */ + fun toRGB(): RGBColorContainer { + val space = ColorSpace.getInstance(ColorSpace.TYPE_YCbCr) + val rgb = space.toRGB(this.ycbcr) + return RGBColorContainer(rgb[0], rgb[1], rgb[2]) + } } \ No newline at end of file diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/cld/CLD.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/cld/CLD.kt index f3b126877..da9d6ca86 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/cld/CLD.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/cld/CLD.kt @@ -156,7 +156,7 @@ class CLD : Analyser { } /* Obtain YCbCr values and calculate DCT. */ - val ycbcrs = Array(3) { IntArray(64) } + val ycbcrs = Array(3) { FloatArray(64) } for (i in 0..63) { val c = RGBColorContainer(rgbs[i]).toYCbCr() ycbcrs[0][i] = c.y @@ -186,11 +186,11 @@ class CLD : Analyser { * @param block Block for which to calculate the DCT. * @return The DCT of the block. */ - private fun dct(block: IntArray): IntArray { + private fun dct(block: FloatArray): FloatArray { var sum: Double var cu: Double var cv: Double - val temp = IntArray(64) + val temp = FloatArray(64) for (u in 0..7) { for (v in 0..7) { @@ -202,7 +202,7 @@ class CLD : Analyser { sum += (block[x * 8 + y] * cos((2 * x + 1) * u * Math.PI / 16.0) * cos((2 * y + 1) * v * Math.PI / 16.0)) } } - temp[SCAN[8 * u + v]] = floor((0.25 * cu * cv * sum) + 0.5).toInt() + temp[SCAN[8 * u + v]] = floor((0.25 * cu * cv * sum) + 0.5).toFloat() } } return temp diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/mediancolor/MedianColor.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/mediancolor/MedianColor.kt index b28ab3be0..3da64d6ae 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/mediancolor/MedianColor.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/mediancolor/MedianColor.kt @@ -136,8 +136,8 @@ class MedianColor : Analyser { b[(rgb.blue * 255).toInt()]++ } - /* Generate vector from per-color histgrams. */ - val lab = RGBColorContainer(medianFromHistogram(r), medianFromHistogram(g), medianFromHistogram(b)).toXYZ().toLab() + /* Generate vector from per-color histograms. */ + val lab = RGBColorContainer(medianFromHistogram(r), medianFromHistogram(g), medianFromHistogram(b)).toLab() return FloatVectorDescriptor(vector = lab.toVector()) } } \ No newline at end of file From 92f62e3598898dc0fae9a7f22c594f050a7a3dcd Mon Sep 17 00:00:00 2001 From: Ralph Gasser Date: Wed, 21 Aug 2024 12:53:54 +0200 Subject: [PATCH 16/20] Minor optimisations. Signed-off-by: Ralph Gasser --- .../core/model/color/RGBColorContainer.kt | 30 ++++++++++++++++++- .../feature/huehistogram/HueHistogram.kt | 3 +- .../feature/mediancolor/MedianColor.kt | 9 +++--- 3 files changed, 36 insertions(+), 6 deletions(-) diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBColorContainer.kt index a1dac81c2..498a6fad9 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBColorContainer.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBColorContainer.kt @@ -45,6 +45,34 @@ value class RGBColorContainer constructor(private val rgb: FloatArray) { val alpha: Float get() = this.rgb[3] + /** + * Converts the [red] component of this [RGBColorContainer] to an [Int] representation. + * + * @return [Int] representation of [blue] color + */ + fun redAsInt(): Int = (this.red * 255).toInt() + + /** + * Converts the [green] component of this [RGBColorContainer] to an [Int] representation. + * + * @return [Int] representation of [blue] color + */ + fun greenAsInt(): Int = (this.green * 255).toInt() + + /** + * Converts the [blue] component of this [RGBColorContainer] to an [Int] representation. + * + * @return [Int] representation of [blue] color + */ + fun blueAsInt(): Int = (this.blue * 255).toInt() + + /** + * Converts the [alpha] component of this [RGBColorContainer] to an [Int] representation. + * + * @return [Int] representation of [alpha] color + */ + fun alphaAsInt(): Int = (this.alpha * 255).toInt() + /** * Adds this [RGBColorContainer] to another [RGBColorContainer]. * @@ -88,7 +116,7 @@ value class RGBColorContainer constructor(private val rgb: FloatArray) { * * @return [Int] representation of RGB color */ - fun toRGBInt(): Int = (this.blue * 255).toInt() and 0XFF or ((this.green * 255).toInt() and 0xFF shl 8) or (((this.red * 255).toInt() and 0xFF) shl 16) + fun toRGBInt(): Int = this.blueAsInt() and 0XFF or ((this.greenAsInt() and 0xFF) shl 8) or ((this.redAsInt() and 0xFF) shl 16) /** * Converts this [RGBColorContainer] to a [HSVColorContainer]. diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogram.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogram.kt index 27ff72eaa..27c8ba374 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogram.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogram.kt @@ -19,6 +19,7 @@ import org.vitrivr.engine.core.model.types.Value import org.vitrivr.engine.core.operators.Operator import org.vitrivr.engine.core.operators.ingest.Extractor import org.vitrivr.engine.core.operators.retrieve.Retriever +import org.vitrivr.engine.core.util.extension.getRGBArray import java.util.* import kotlin.reflect.KClass @@ -128,7 +129,7 @@ class HueHistogram : Analyser { fun analyse(content: ImageContent): FloatVectorDescriptor { /* Generate histogram. */ val hist = FloatArray(VECTOR_SIZE) - val colors = content.content.getRGB(0, 0, content.content.width, content.content.height, null, 0, content.content.width) + val colors = content.content.getRGBArray() for (color in colors) { val container: HSVColorContainer = RGBColorContainer(color).toHSV() if (container.saturation > 0.2f && container.value > 0.3f) { diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/mediancolor/MedianColor.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/mediancolor/MedianColor.kt index 3da64d6ae..b95507a00 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/mediancolor/MedianColor.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/mediancolor/MedianColor.kt @@ -17,6 +17,7 @@ import org.vitrivr.engine.core.model.types.Value import org.vitrivr.engine.core.operators.Operator import org.vitrivr.engine.core.operators.ingest.Extractor import org.vitrivr.engine.core.operators.retrieve.Retriever +import org.vitrivr.engine.core.util.extension.getRGBArray import org.vitrivr.engine.core.util.math.StatisticsHelper.medianFromHistogram import java.util.* @@ -128,12 +129,12 @@ class MedianColor : Analyser { val b = IntArray(256) /* Extract colors from content and generate histogram. */ - val colors: IntArray = content.content.getRGB(0, 0, content.content.width, content.content.height, null, 0, content.content.width); + val colors: IntArray = content.content.getRGBArray() for (color in colors) { val rgb = RGBColorContainer(color) - r[(rgb.red * 255).toInt()]++ - g[(rgb.green * 255).toInt()]++ - b[(rgb.blue * 255).toInt()]++ + r[rgb.redAsInt()]++ + g[rgb.greenAsInt()]++ + b[rgb.blueAsInt()]++ } /* Generate vector from per-color histograms. */ From 4883d0e6bc9436015bee06800014416c5cf18be5 Mon Sep 17 00:00:00 2001 From: faberf Date: Fri, 23 Aug 2024 13:29:27 +0200 Subject: [PATCH 17/20] implemented retrieverfromcontent --- .../classification/ImageClassification.kt | 26 +++++++++++++++---- .../ImageClassificationExtractor.kt | 9 ++++--- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassification.kt b/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassification.kt index fb3ecc4b8..d4f953ab5 100644 --- a/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassification.kt +++ b/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassification.kt @@ -5,8 +5,10 @@ import org.vitrivr.engine.base.features.external.common.ExternalFesAnalyser import org.vitrivr.engine.core.context.IndexContext import org.vitrivr.engine.core.context.QueryContext import org.vitrivr.engine.core.features.AbstractRetriever +import org.vitrivr.engine.core.model.content.Content import org.vitrivr.engine.core.model.content.element.ContentElement import org.vitrivr.engine.core.model.content.element.ImageContent +import org.vitrivr.engine.core.model.content.element.TextContent import org.vitrivr.engine.core.model.descriptor.struct.LabelDescriptor import org.vitrivr.engine.core.model.metamodel.Analyser.Companion.merge import org.vitrivr.engine.core.model.metamodel.Schema @@ -25,14 +27,14 @@ import java.util.* * @author Fynn Faber * @version 1.0.0 */ -class ImageClassification : ExternalFesAnalyser() { +class ImageClassification : ExternalFesAnalyser, LabelDescriptor>() { companion object{ const val CLASSES_PARAMETER_NAME = "classes" const val THRESHOLD_PARAMETER_NAME = "threshold" const val TOPK_PARAMETER_NAME = "top_k" } - override val contentClasses = setOf(ImageContent::class) + override val contentClasses = setOf(ImageContent::class, TextContent::class) override val descriptorClass = LabelDescriptor::class /** @@ -63,11 +65,25 @@ class ImageClassification : ExternalFesAnalyser() * @param context The [IndexContext] to use with the [ImageClassification]. * @return [ImageClassification] */ - override fun newExtractor(field: Schema.Field, input: Operator, context: IndexContext) = ImageClassificationExtractor(input, field, this, merge(field, context)) + override fun newExtractor(field: Schema.Field, LabelDescriptor>, input: Operator, context: IndexContext) = ImageClassificationExtractor(input, field, this, merge(field, context)) - override fun newRetrieverForQuery(field: Schema.Field, query: Query, context: QueryContext): Retriever { + override fun newRetrieverForQuery(field: Schema.Field, LabelDescriptor>, query: Query, context: QueryContext): Retriever, LabelDescriptor> { require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } require(query is SimpleBooleanQuery<*>) { "The query is not a boolean query. This is a programmer's error!" } - return object : AbstractRetriever(field, query, context){} + return object : AbstractRetriever, LabelDescriptor>(field, query, context){} + } + + override fun newRetrieverForContent( + field: Schema.Field, LabelDescriptor>, + content: Collection>, + context: QueryContext + ): Retriever, LabelDescriptor> { + val firstContent = content.first() + if (content.size != 1 || firstContent !is TextContent) { + throw IllegalArgumentException("The content does not match the expected type. This is a programmer's error!") + } + val query = SimpleBooleanQuery(value = Value.String(firstContent.content), attributeName = "label") + return newRetrieverForQuery(field, query, context) + } } \ No newline at end of file diff --git a/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassificationExtractor.kt b/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassificationExtractor.kt index 9684ef944..0e5057377 100644 --- a/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassificationExtractor.kt +++ b/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassificationExtractor.kt @@ -6,6 +6,7 @@ import org.vitrivr.engine.base.features.external.common.FesExtractor import org.vitrivr.engine.base.features.external.implementations.classification.ImageClassification.Companion.CLASSES_PARAMETER_NAME import org.vitrivr.engine.base.features.external.implementations.classification.ImageClassification.Companion.THRESHOLD_PARAMETER_NAME import org.vitrivr.engine.base.features.external.implementations.classification.ImageClassification.Companion.TOPK_PARAMETER_NAME +import org.vitrivr.engine.core.model.content.element.ContentElement import org.vitrivr.engine.core.model.content.element.ImageContent import org.vitrivr.engine.core.model.descriptor.struct.LabelDescriptor import org.vitrivr.engine.core.model.metamodel.Schema @@ -21,10 +22,10 @@ import java.util.* */ class ImageClassificationExtractor( input: Operator, - field: Schema.Field?, - analyser: ExternalFesAnalyser, + field: Schema.Field, LabelDescriptor>?, + analyser: ExternalFesAnalyser, LabelDescriptor>, parameters: Map -) : FesExtractor(input, field, analyser, parameters) { +) : FesExtractor, LabelDescriptor>(input, field, analyser, parameters) { /** The [ZeroShotClassificationApi] used to perform extraction with. */ @@ -48,7 +49,7 @@ class ImageClassificationExtractor( val flatResults = this.api.analyseBatched( retrievables.flatMap { - this.filterContent(it).map { it to classes } + this.filterContent(it).filterIsInstance().map { it to classes } }).mapIndexed { idx, result -> result.mapIndexed { idy, confidence -> LabelDescriptor( From 0e2c7ded255c861aedc21523eff051f837c417c2 Mon Sep 17 00:00:00 2001 From: faberf Date: Fri, 23 Aug 2024 13:48:20 +0200 Subject: [PATCH 18/20] fixed issue with classifier --- .../classification/ImageClassificationExtractor.kt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassificationExtractor.kt b/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassificationExtractor.kt index 0e5057377..44155f7f0 100644 --- a/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassificationExtractor.kt +++ b/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassificationExtractor.kt @@ -47,10 +47,11 @@ class ImageClassificationExtractor( val topK = this.parameters[TOPK_PARAMETER_NAME]?.toInt() ?: 1 val threshold = this.parameters[THRESHOLD_PARAMETER_NAME]?.toFloat() ?: 0.0f - val flatResults = this.api.analyseBatched( - retrievables.flatMap { - this.filterContent(it).filterIsInstance().map { it to classes } - }).mapIndexed { idx, result -> + val content = retrievables.mapIndexed { idx, retrievable -> + this.filterContent(retrievable).filterIsInstance().map { idx to (it to classes) } + }.flatten() + + return this.api.analyseBatched(content.map{it.second}).zip(content.map{it.first}).map { (result, idx) -> result.mapIndexed { idy, confidence -> LabelDescriptor( UUID.randomUUID(), @@ -63,6 +64,5 @@ class ImageClassificationExtractor( ) }.filter { it.confidence.value >= threshold }.sortedByDescending { it.confidence.value }.take(topK) } - return flatResults } } \ No newline at end of file From 11901cb2c5f95198c2e59a4ffd1b5441ab207c6c Mon Sep 17 00:00:00 2001 From: Ralph Gasser Date: Fri, 23 Aug 2024 15:13:30 +0200 Subject: [PATCH 19/20] Minor adjustment plus added missing implementation. Signed-off-by: Ralph Gasser --- .../classification/ImageClassification.kt | 45 ++++++++++++++++--- 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassification.kt b/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassification.kt index d4f953ab5..70f0fac3e 100644 --- a/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassification.kt +++ b/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/classification/ImageClassification.kt @@ -10,10 +10,13 @@ import org.vitrivr.engine.core.model.content.element.ContentElement import org.vitrivr.engine.core.model.content.element.ImageContent import org.vitrivr.engine.core.model.content.element.TextContent import org.vitrivr.engine.core.model.descriptor.struct.LabelDescriptor +import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor +import org.vitrivr.engine.core.model.metamodel.Analyser import org.vitrivr.engine.core.model.metamodel.Analyser.Companion.merge import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.query.Query import org.vitrivr.engine.core.model.query.bool.SimpleBooleanQuery +import org.vitrivr.engine.core.model.query.proximity.ProximityQuery import org.vitrivr.engine.core.model.retrievable.Retrievable import org.vitrivr.engine.core.model.types.Value import org.vitrivr.engine.core.operators.Operator @@ -67,23 +70,53 @@ class ImageClassification : ExternalFesAnalyser, LabelDescript */ override fun newExtractor(field: Schema.Field, LabelDescriptor>, input: Operator, context: IndexContext) = ImageClassificationExtractor(input, field, this, merge(field, context)) + /** + * Generates and returns a new [Retriever] instance for this [ImageClassification]. + * + * @param field The [Schema.Field] to create an [Retriever] for. + * @param query The [Query] to use with the [Retriever]. + * @param context The [QueryContext] to use with the [Retriever]. + * + * @return A new [Retriever] instance for this [Analyser] + */ override fun newRetrieverForQuery(field: Schema.Field, LabelDescriptor>, query: Query, context: QueryContext): Retriever, LabelDescriptor> { require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } require(query is SimpleBooleanQuery<*>) { "The query is not a boolean query. This is a programmer's error!" } return object : AbstractRetriever, LabelDescriptor>(field, query, context){} } + /** + * Generates and returns a new [Retriever] instance for this [ImageClassification]. + * + * Invoking this method involves converting the provided [FloatVectorDescriptor] into a [ProximityQuery] that can be used to retrieve similar [ImageContent] elements. + * + * @param field The [Schema.Field] to create an [Retriever] for. + * @param descriptors An array of [FloatVectorDescriptor] elements to use with the [Retriever] + * @param context The [QueryContext] to use with the [Retriever] + */ + override fun newRetrieverForDescriptors(field: Schema.Field, LabelDescriptor>, descriptors: Collection, context: QueryContext): Retriever, LabelDescriptor> { + val descriptor = descriptors.firstOrNull()?.label ?: throw IllegalArgumentException("No label descriptor provided.") + val query = SimpleBooleanQuery(value = descriptor, attributeName = "label") + return newRetrieverForQuery(field, query, context) + } + + /** + * Generates and returns a new [Retriever] instance for this [ImageClassification]. + * + * Invoking this method involves converting the provided [ImageContent] and the [QueryContext] into a [FloatVectorDescriptor] + * that can be used to retrieve similar [ImageContent] elements. + * + * @param field The [Schema.Field] to create an [Retriever] for. + * @param content An array of [Content] elements to use with the [Retriever] + * @param context The [QueryContext] to use with the [Retriever] + */ override fun newRetrieverForContent( field: Schema.Field, LabelDescriptor>, content: Collection>, context: QueryContext ): Retriever, LabelDescriptor> { - val firstContent = content.first() - if (content.size != 1 || firstContent !is TextContent) { - throw IllegalArgumentException("The content does not match the expected type. This is a programmer's error!") - } - val query = SimpleBooleanQuery(value = Value.String(firstContent.content), attributeName = "label") + val first = content.filterIsInstance().firstOrNull() ?: throw IllegalArgumentException("The content does not contain any text.") + val query = SimpleBooleanQuery(value = Value.String(first.content), attributeName = "label") return newRetrieverForQuery(field, query, context) - } } \ No newline at end of file From a60e978774d124f573b31cca7656c64dec5d54c5 Mon Sep 17 00:00:00 2001 From: Ralph Gasser Date: Mon, 26 Aug 2024 10:27:49 +0200 Subject: [PATCH 20/20] Cleans and updates dependencies. --- build.gradle | 4 +- gradle.properties | 37 ++++++------- vitrivr-engine-core/build.gradle | 2 +- .../engine/plugin/cottontaildb/Sandbox.kt | 52 ------------------- .../descriptors/AbstractDescriptorReader.kt | 7 +-- .../descriptors/CottontailDescriptorWriter.kt | 2 +- .../retrievable/RetrievableReader.kt | 10 ++-- .../retrievable/RetrievableWriter.kt | 3 +- vitrivr-engine-module-fes/build.gradle | 9 +--- vitrivr-engine-server/build.gradle | 4 +- 10 files changed, 36 insertions(+), 94 deletions(-) delete mode 100644 vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/Sandbox.kt diff --git a/build.gradle b/build.gradle index e1d102112..31bcb8c51 100644 --- a/build.gradle +++ b/build.gradle @@ -3,8 +3,8 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinCompile plugins { id 'org.jetbrains.kotlin.jvm' version "$version_kotlin" id 'org.jetbrains.kotlin.plugin.serialization' version "$version_kotlin" - id 'org.openapi.generator' version '7.4.0' - id "de.undercouch.download" version "5.4.0" + id 'org.openapi.generator' version '7.8.0' + id 'de.undercouch.download' version "5.6.0" id 'java-library' id 'idea' } diff --git a/gradle.properties b/gradle.properties index 90b5f8205..a66a1f5ec 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,29 +1,26 @@ version_boofcv=1.1.5 version_caffeine=3.1.8 -version_clikt=4.2.0 +version_clikt=4.2.2 version_commonsmath3=3.6.1 -version_cottontaildb=0.16.1 -version_javacv=1.5.9 -version_javalin=6.1.3 -version_javalinopenapi=6.1.3 -version_javalinssl=6.1.3 -version_jdbc_postgres=42.7.3 -version_jline=3.23.0 -version_junit=5.10.1 -version_junit_platform=1.10.1 -version_grpc=1.60.0 -version_kotlin=1.9.21 +version_cottontaildb=0.16.7 +version_javacv=1.5.10 +version_javalin=6.3.0 +version_jdbc_postgres=42.7.4 +version_jline=3.26.3 +version_junit=5.11.0 +version_junit_platform=1.11.0 +version_grpc=1.66.0 +version_kotlin=1.9.25 version_kotlinx_coroutines=1.7.3 version_kotlinx_serialization=1.6.2 -version_kotlinx_datetime=0.4.1 -version_kotlinlogging = 5.1.0 -version_log4j2=2.20.0 +version_kotlinx_datetime=0.6.1 +version_kotlinlogging = 7.0.0 +version_log4j2=2.23.1 version_metadataextractor=2.19.0 -version_okhttp3=4.12.0 version_picnic=0.7.0 -version_protobuf=3.25.1 -version_scrimage=4.1.1 -version_slf4j=2.0.9 +version_protobuf=4.27.3 +version_scrimage=4.2.0 +version_slf4j=2.0.16 version_jogl=2.3.2 version_joml=1.9.25 -version_ktor=2.3.6 +version_ktor=2.3.12 diff --git a/vitrivr-engine-core/build.gradle b/vitrivr-engine-core/build.gradle index c8093c8e9..a456d15fc 100755 --- a/vitrivr-engine-core/build.gradle +++ b/vitrivr-engine-core/build.gradle @@ -10,7 +10,7 @@ dependencies { /** dependencies for exif metadata extraction. */ implementation group: 'com.drewnoakes', name: 'metadata-extractor', version: version_metadataextractor - implementation group: 'io.javalin.community.openapi', name: 'javalin-openapi-plugin', version: version_javalinopenapi + implementation group: 'io.javalin.community.openapi', name: 'javalin-openapi-plugin', version: version_javalin /* Test Fixtures from Cottontail DB core. .*/ testFixturesImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-api', version: version_junit diff --git a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/Sandbox.kt b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/Sandbox.kt deleted file mode 100644 index 2e7c6604a..000000000 --- a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/Sandbox.kt +++ /dev/null @@ -1,52 +0,0 @@ -package org.vitrivr.engine.plugin.cottontaildb - -import io.grpc.ManagedChannel -import io.grpc.ManagedChannelBuilder -import org.vitrivr.cottontail.client.SimpleClient -import org.vitrivr.cottontail.client.language.basics.Direction -import org.vitrivr.cottontail.client.language.basics.Distances -import org.vitrivr.cottontail.client.language.basics.expression.Column -import org.vitrivr.cottontail.client.language.basics.expression.Literal -import org.vitrivr.cottontail.client.language.basics.predicate.Compare -import org.vitrivr.cottontail.client.language.dql.Query -import org.vitrivr.cottontail.core.database.Name -import org.vitrivr.cottontail.core.values.StringValue -import org.vitrivr.cottontail.core.values.UuidValue -import java.util.* - -object Sandbox { - - @JvmStatic - fun main(args: Array) { - - val channel: ManagedChannel = ManagedChannelBuilder.forAddress("127.0.0.1", 1865) - .enableFullStreamDecompression() - .usePlaintext() - .build() - - val client = SimpleClient(channel) - - - val query = Query(Name.EntityName.create("mvk", "descriptor_file")) - .select("*") - .where(Compare( - Column(Name.ColumnName.create("mvk", "descriptor_file", "path")), - Compare.Operator.EQUAL, - Literal(StringValue("Z:\\Datasets\\MarineVideoKit\\videos-optimized\\Tulamben2_Jun2022_0016.mp4"))), - ).limit(1000) - - - val query1 = Query(Name.EntityName.create("mvk-2", "retrievable")) - .select("*") - .where(Compare( - Column(Name.ColumnName.create("mvk-2", "retrievable", "retrievableid")), - Compare.Operator.EQUAL, - Literal(UuidValue(UUID.fromString("8ce4920f-a230-4a6d-8186-20dc029a03af")))), - ).limit(1000) - - val results = client.query(query1).forEach { - println(it) - } - } - -} diff --git a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/AbstractDescriptorReader.kt b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/AbstractDescriptorReader.kt index 6f4b36ffc..1358fc5df 100644 --- a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/AbstractDescriptorReader.kt +++ b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/AbstractDescriptorReader.kt @@ -3,6 +3,7 @@ package org.vitrivr.engine.plugin.cottontaildb.descriptors import io.grpc.StatusRuntimeException import org.vitrivr.cottontail.client.language.basics.expression.Column import org.vitrivr.cottontail.client.language.basics.expression.Literal +import org.vitrivr.cottontail.client.language.basics.expression.ValueList import org.vitrivr.cottontail.client.language.basics.predicate.Compare import org.vitrivr.cottontail.core.database.Name import org.vitrivr.cottontail.core.tuple.Tuple @@ -111,7 +112,7 @@ abstract class AbstractDescriptorReader>(final override val fi */ override fun getAll(descriptorIds: Iterable): Sequence { val query = org.vitrivr.cottontail.client.language.dql.Query(this.entityName) - .where(Compare(Column(this.entityName.column(DESCRIPTOR_ID_COLUMN_NAME)), Compare.Operator.IN, org.vitrivr.cottontail.client.language.basics.expression.List(descriptorIds.map { UuidValue(it) }.toTypedArray()))) + .where(Compare(Column(this.entityName.column(DESCRIPTOR_ID_COLUMN_NAME)), Compare.Operator.IN, ValueList(descriptorIds.map { UuidValue(it) }.toTypedArray()))) return try { val result = this.connection.client.query(query) result.asSequence().map { this.tupleToDescriptor(it) } @@ -129,7 +130,7 @@ abstract class AbstractDescriptorReader>(final override val fi */ override fun getAllForRetrievable(retrievableIds: Iterable): Sequence { val query = org.vitrivr.cottontail.client.language.dql.Query(this.entityName) - .where(Compare(Column(this.entityName.column(RETRIEVABLE_ID_COLUMN_NAME)), Compare.Operator.IN, org.vitrivr.cottontail.client.language.basics.expression.List(retrievableIds.map { UuidValue(it) }.toTypedArray()))) + .where(Compare(Column(this.entityName.column(RETRIEVABLE_ID_COLUMN_NAME)), Compare.Operator.IN, ValueList(retrievableIds.map { UuidValue(it) }.toTypedArray()))) return try { val result = this.connection.client.query(query) result.asSequence().map { this.tupleToDescriptor(it) } @@ -199,7 +200,7 @@ abstract class AbstractDescriptorReader>(final override val fi Compare( Column(entityName.column(RETRIEVABLE_ID_COLUMN_NAME)), Compare.Operator.IN, - org.vitrivr.cottontail.client.language.basics.expression.List(ids.map { UuidValue(it) }.toTypedArray()) + ValueList(ids.map { UuidValue(it) }.toTypedArray()) ) ) diff --git a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/CottontailDescriptorWriter.kt b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/CottontailDescriptorWriter.kt index e7527e181..9b065ca86 100644 --- a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/CottontailDescriptorWriter.kt +++ b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/CottontailDescriptorWriter.kt @@ -167,7 +167,7 @@ open class CottontailDescriptorWriter>(final override val fiel Compare( Column(this.entityName.column(DESCRIPTOR_ID_COLUMN_NAME)), Compare.Operator.IN, - org.vitrivr.cottontail.client.language.basics.expression.List(ids.toTypedArray()) + org.vitrivr.cottontail.client.language.basics.expression.ValueList(ids.toTypedArray()) ) ) diff --git a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/retrievable/RetrievableReader.kt b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/retrievable/RetrievableReader.kt index 45bba8690..b67e02394 100644 --- a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/retrievable/RetrievableReader.kt +++ b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/retrievable/RetrievableReader.kt @@ -4,8 +4,8 @@ import io.github.oshai.kotlinlogging.KLogger import io.github.oshai.kotlinlogging.KotlinLogging import io.grpc.StatusRuntimeException import org.vitrivr.cottontail.client.language.basics.expression.Column -import org.vitrivr.cottontail.client.language.basics.expression.List import org.vitrivr.cottontail.client.language.basics.expression.Literal +import org.vitrivr.cottontail.client.language.basics.expression.ValueList import org.vitrivr.cottontail.client.language.basics.predicate.And import org.vitrivr.cottontail.client.language.basics.predicate.Compare import org.vitrivr.cottontail.client.language.dql.Query @@ -103,7 +103,7 @@ internal class RetrievableReader(override val connection: CottontailConnection) Compare( Column(this.entityName.column(RETRIEVABLE_ID_COLUMN_NAME)), Compare.Operator.IN, - List(ids.map { UuidValue(it) }.toTypedArray()) + ValueList(ids.map { UuidValue(it) }.toTypedArray()) ) ) return try { @@ -160,7 +160,7 @@ internal class RetrievableReader(override val connection: CottontailConnection) Compare( Column(Name.ColumnName.create(SUBJECT_ID_COLUMN_NAME)), Compare.Operator.IN, - List(subjectIds.map { UuidValue(it) }.toTypedArray()) + ValueList(subjectIds.map { UuidValue(it) }.toTypedArray()) ) } else { null @@ -169,7 +169,7 @@ internal class RetrievableReader(override val connection: CottontailConnection) Compare( Column(Name.ColumnName.create(PREDICATE_COLUMN_NAME)), Compare.Operator.IN, - List(predicates.map { StringValue(it) }.toTypedArray()) + ValueList(predicates.map { StringValue(it) }.toTypedArray()) ) } else { null @@ -178,7 +178,7 @@ internal class RetrievableReader(override val connection: CottontailConnection) Compare( Column(Name.ColumnName.create(OBJECT_ID_COLUMN_NAME)), Compare.Operator.IN, - List(objectIds.map { UuidValue(it) }.toTypedArray()) + ValueList(objectIds.map { UuidValue(it) }.toTypedArray()) ) } else { null diff --git a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/retrievable/RetrievableWriter.kt b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/retrievable/RetrievableWriter.kt index c8145ab17..4ea6a9858 100644 --- a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/retrievable/RetrievableWriter.kt +++ b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/retrievable/RetrievableWriter.kt @@ -5,6 +5,7 @@ import io.github.oshai.kotlinlogging.KotlinLogging import io.grpc.StatusRuntimeException import org.vitrivr.cottontail.client.language.basics.expression.Column import org.vitrivr.cottontail.client.language.basics.expression.Literal +import org.vitrivr.cottontail.client.language.basics.expression.ValueList import org.vitrivr.cottontail.client.language.basics.predicate.And import org.vitrivr.cottontail.client.language.basics.predicate.Compare import org.vitrivr.cottontail.client.language.dml.BatchInsert @@ -142,7 +143,7 @@ internal class RetrievableWriter(override val connection: CottontailConnection) Compare( Column(this.entityName.column(RETRIEVABLE_ID_COLUMN_NAME)), Compare.Operator.IN, - org.vitrivr.cottontail.client.language.basics.expression.List(ids.toTypedArray()) + ValueList(ids.toTypedArray()) ) ) diff --git a/vitrivr-engine-module-fes/build.gradle b/vitrivr-engine-module-fes/build.gradle index 2fb0683e1..2d37166c5 100644 --- a/vitrivr-engine-module-fes/build.gradle +++ b/vitrivr-engine-module-fes/build.gradle @@ -50,17 +50,12 @@ dependencies { /* vitrivr engine features dependency. */ implementation project(path: ':vitrivr-engine-core') - /** OKHttp3 Client */ - implementation group: 'com.squareup.okhttp3', name: 'okhttp', version: version_okhttp3 - implementation group: 'com.squareup.okhttp3', name: 'logging-interceptor', version: version_okhttp3 - - implementation group: 'org.jetbrains.kotlinx', name: 'kotlinx-coroutines-core', version: version_kotlinx_coroutines - implementation group: 'org.jetbrains.kotlinx', name: 'kotlinx-serialization-json', version: version_kotlinx_serialization - + /* Ktor client. */ implementation group: 'io.ktor', name: 'ktor-client-content-negotiation', version: version_ktor implementation group: 'io.ktor', name: 'ktor-serialization-kotlinx-json', version: version_ktor implementation group: 'io.ktor', name: 'ktor-client-core', version: version_ktor implementation group: 'io.ktor', name: 'ktor-client-okhttp', version: version_ktor + /* Kotlinx */ implementation group: 'org.jetbrains.kotlinx', name: 'kotlinx-datetime', version: version_kotlinx_datetime } diff --git a/vitrivr-engine-server/build.gradle b/vitrivr-engine-server/build.gradle index 007186c30..654a8ca13 100644 --- a/vitrivr-engine-server/build.gradle +++ b/vitrivr-engine-server/build.gradle @@ -21,8 +21,8 @@ dependencies { /** Javalin */ implementation group: 'io.javalin', name: 'javalin', version: version_javalin - implementation group: 'io.javalin.community.openapi', name: 'javalin-openapi-plugin', version: version_javalinopenapi - implementation group: 'io.javalin.community.openapi', name: 'javalin-swagger-plugin', version: version_javalinssl + implementation group: 'io.javalin.community.openapi', name: 'javalin-openapi-plugin', version: version_javalin + implementation group: 'io.javalin.community.openapi', name: 'javalin-swagger-plugin', version: version_javalin implementation group: 'io.javalin.community.ssl', name: 'ssl-plugin', version: version_javalin /* Kapt. */