diff --git a/gradle.properties b/gradle.properties index cf611c897..90b5f8205 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,4 @@ +version_boofcv=1.1.5 version_caffeine=3.1.8 version_clikt=4.2.0 version_commonsmath3=3.6.1 diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/BlackholeConnectionProvider.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/BlackholeConnectionProvider.kt index f6837bfa1..0aec544b4 100755 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/BlackholeConnectionProvider.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/BlackholeConnectionProvider.kt @@ -16,6 +16,8 @@ class BlackholeConnectionProvider : ConnectionProvider { override val databaseName: String = "blackhole" override val version: String = "1.0.0" override fun openConnection(schemaName: String, parameters: Map) = BlackholeConnection(schemaName, this, parameters["log"]?.toBoolean() == true) - override fun > register(descriptorClass: KClass, provider: DescriptorProvider<*>) { /* No op. */ } + override fun > register(descriptorClass: KClass, provider: DescriptorProvider<*>) { /* No op. */ + } + override fun > obtain(descriptorClass: KClass) = BlackholeDescriptionProvider() } diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/descriptors/BlackholeDescriptionProvider.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/descriptors/BlackholeDescriptionProvider.kt index 98f072e57..bc39c9a74 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/descriptors/BlackholeDescriptionProvider.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/descriptors/BlackholeDescriptionProvider.kt @@ -14,8 +14,8 @@ import org.vitrivr.engine.core.model.metamodel.Schema * @author Ralph Gasser * @version 1.0.0 */ -class BlackholeDescriptionProvider>: DescriptorProvider { - override fun newInitializer(connection: Connection, field: Schema.Field<*, T>)= BlackholeDescriptorInitializer(connection as BlackholeConnection, field) +class BlackholeDescriptionProvider> : DescriptorProvider { + override fun newInitializer(connection: Connection, field: Schema.Field<*, T>) = BlackholeDescriptorInitializer(connection as BlackholeConnection, field) override fun newReader(connection: Connection, field: Schema.Field<*, T>): DescriptorReader = BlackholeDescriptorReader(connection as BlackholeConnection, field) override fun newWriter(connection: Connection, field: Schema.Field<*, T>): DescriptorWriter = BlackholeDescriptorWriter(connection as BlackholeConnection, field) } \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/descriptors/BlackholeDescriptorInitializer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/descriptors/BlackholeDescriptorInitializer.kt index b8b1a973e..b8aef87de 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/descriptors/BlackholeDescriptorInitializer.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/descriptors/BlackholeDescriptorInitializer.kt @@ -11,7 +11,7 @@ import org.vitrivr.engine.core.model.metamodel.Schema * @author Ralph Gasser * @version 1.0.0 */ -class BlackholeDescriptorInitializer>(private val connection: BlackholeConnection, override val field: Schema.Field<*, T>): DescriptorInitializer { +class BlackholeDescriptorInitializer>(private val connection: BlackholeConnection, override val field: Schema.Field<*, T>) : DescriptorInitializer { override fun initialize() = this.connection.logIf("Initializing descriptor entity '${this.field.fieldName}'.") override fun deinitialize() = this.connection.logIf("De-initializing descriptor entity '${this.field.fieldName}'.") override fun isInitialized(): Boolean = false diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/descriptors/BlackholeDescriptorReader.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/descriptors/BlackholeDescriptorReader.kt index 707cd70a1..6c14a25c6 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/descriptors/BlackholeDescriptorReader.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/descriptors/BlackholeDescriptorReader.kt @@ -15,7 +15,7 @@ import org.vitrivr.engine.core.model.retrievable.Retrieved * @author Ralph Gasser * @version 1.0.0 */ -class BlackholeDescriptorReader>(override val connection: BlackholeConnection, override val field: Schema.Field<*, T>) : DescriptorReader { +class BlackholeDescriptorReader>(override val connection: BlackholeConnection, override val field: Schema.Field<*, T>) : DescriptorReader { override fun exists(descriptorId: DescriptorId): Boolean = false override fun get(descriptorId: DescriptorId): T? = null override fun getAll(descriptorIds: Iterable) = emptySequence() diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/descriptors/BlackholeDescriptorWriter.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/descriptors/BlackholeDescriptorWriter.kt index f1414f9d3..04bf5bb18 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/descriptors/BlackholeDescriptorWriter.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/descriptors/BlackholeDescriptorWriter.kt @@ -11,7 +11,7 @@ import org.vitrivr.engine.core.model.metamodel.Schema * @author Ralph Gasser * @version 1.0.0 */ -class BlackholeDescriptorWriter>(override val connection: BlackholeConnection, override val field: Schema.Field<*, T>): DescriptorWriter { +class BlackholeDescriptorWriter>(override val connection: BlackholeConnection, override val field: Schema.Field<*, T>) : DescriptorWriter { override fun add(item: T): Boolean { this.connection.logIf("Adding descriptor '${item.id}' to entity '${this.field.fieldName}'.") return false diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/retrievable/BlackholeRetrievableInitializer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/retrievable/BlackholeRetrievableInitializer.kt index cbc2b06c7..4bde3b4b1 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/retrievable/BlackholeRetrievableInitializer.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/retrievable/BlackholeRetrievableInitializer.kt @@ -9,7 +9,7 @@ import org.vitrivr.engine.core.database.retrievable.RetrievableInitializer * @author Ralph Gasser * @version 1.0.0 */ -class BlackholeRetrievableInitializer(private val connection: BlackholeConnection): RetrievableInitializer { +class BlackholeRetrievableInitializer(private val connection: BlackholeConnection) : RetrievableInitializer { override fun initialize() = this.connection.logIf("Initializing entities 'retrievable' and 'relationship'.") override fun deinitialize() = this.connection.logIf("De-initializing entities 'retrievable' and 'relationship'.") override fun isInitialized(): Boolean = false diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/retrievable/BlackholeRetrievableReader.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/retrievable/BlackholeRetrievableReader.kt index d4e287078..256942f24 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/retrievable/BlackholeRetrievableReader.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/retrievable/BlackholeRetrievableReader.kt @@ -14,7 +14,7 @@ import org.vitrivr.engine.core.model.retrievable.RetrievableId */ class BlackholeRetrievableReader(override val connection: BlackholeConnection) : RetrievableReader { override fun get(id: RetrievableId): Retrievable? = null - override fun exists(id: RetrievableId): Boolean = false + override fun exists(id: RetrievableId): Boolean = false override fun getAll(ids: Iterable) = emptySequence() override fun getAll() = emptySequence() override fun getConnections(subjectIds: Collection, predicates: Collection, objectIds: Collection) = emptySequence>() diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/retrievable/BlackholeRetrievableWriter.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/retrievable/BlackholeRetrievableWriter.kt index 26888832f..38695d7a6 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/retrievable/BlackholeRetrievableWriter.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/retrievable/BlackholeRetrievableWriter.kt @@ -12,14 +12,14 @@ import org.vitrivr.engine.core.model.retrievable.Retrievable * @author Ralph Gasser * @version 1.0.0 */ -class BlackholeRetrievableWriter(override val connection: BlackholeConnection): RetrievableWriter { +class BlackholeRetrievableWriter(override val connection: BlackholeConnection) : RetrievableWriter { override fun connect(relationship: Relationship): Boolean { this.connection.logIf("Adding relationship ${relationship.subjectId} >[${relationship.predicate}] ${relationship.objectId}.") return false } override fun connectAll(relationships: Iterable): Boolean { - relationships.forEach { relationship -> this.connection.logIf("Adding relationship ${relationship.subjectId} >[${relationship.predicate}] ${relationship.objectId}.")} + relationships.forEach { relationship -> this.connection.logIf("Adding relationship ${relationship.subjectId} >[${relationship.predicate}] ${relationship.objectId}.") } return false } @@ -29,7 +29,7 @@ class BlackholeRetrievableWriter(override val connection: BlackholeConnection): } override fun disconnectAll(relationships: Iterable): Boolean { - relationships.forEach { relationship -> this.connection.logIf("Removing relationship ${relationship.subjectId} >[${relationship.predicate}] ${relationship.objectId}.")} + relationships.forEach { relationship -> this.connection.logIf("Removing relationship ${relationship.subjectId} >[${relationship.predicate}] ${relationship.objectId}.") } return false } diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/AbstractRetriever.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/AbstractRetriever.kt index 859523f55..6d7324d1a 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/AbstractRetriever.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/AbstractRetriever.kt @@ -1,5 +1,7 @@ package org.vitrivr.engine.core.features +import io.github.oshai.kotlinlogging.KLogger +import io.github.oshai.kotlinlogging.KotlinLogging import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.flow @@ -20,13 +22,16 @@ import org.vitrivr.engine.core.operators.retrieve.Retriever * @see [AbstractRetriever] * * @author Rahel Arnold - * @version 1.0.0 + * @version 1.0.1 */ abstract class AbstractRetriever, D : Descriptor<*>>(override val field: Schema.Field, val query: Query, val context: QueryContext) : Retriever { /** The [DescriptorReader] instance used by this [AbstractRetriever]. */ protected val reader: DescriptorReader by lazy { this.field.getReader() } + /** The [KLogger] instance used by this [AbstractRetriever]. */ + protected val logger: KLogger = KotlinLogging.logger {} + /** * Simplest implementation of the retrieval logic simply hand the [Query] to the [DescriptorReader] and emit the results. * diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColor.kt index dfe802759..fc307eaf0 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColor.kt @@ -2,9 +2,9 @@ package org.vitrivr.engine.core.features.averagecolor import org.vitrivr.engine.core.context.IndexContext import org.vitrivr.engine.core.context.QueryContext -import org.vitrivr.engine.core.model.color.MutableRGBFloatColorContainer -import org.vitrivr.engine.core.model.color.RGBByteColorContainer -import org.vitrivr.engine.core.model.color.RGBFloatColorContainer +import org.vitrivr.engine.core.features.dense.DenseRetriever +import org.vitrivr.engine.core.math.correspondence.LinearCorrespondence +import org.vitrivr.engine.core.model.color.RGBColorContainer import org.vitrivr.engine.core.model.content.Content import org.vitrivr.engine.core.model.content.element.ImageContent import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor @@ -30,28 +30,9 @@ import java.util.* * @version 1.0.0 */ class AverageColor : Analyser { - override val contentClasses = setOf(ImageContent::class) override val descriptorClass = FloatVectorDescriptor::class - companion object { - /** - * Performs the [AverageColor] analysis on the provided [List] of [ImageContent] elements. - * - * @param content The [List] of [ImageContent] elements. - * @return [List] of [FloatVectorDescriptor]s. - */ - fun analyse(content: Collection): List = content.map { - val color = MutableRGBFloatColorContainer() - val rgb = it.content.getRGBArray() - rgb.forEach { c -> color += RGBByteColorContainer(c) } - - /* Generate descriptor. */ - val averageColor = RGBFloatColorContainer(color.red / rgb.size, color.green / rgb.size, color.blue / rgb.size) - FloatVectorDescriptor(UUID.randomUUID(), null, averageColor.toVector()) - } - } - /** * Generates a prototypical [FloatVectorDescriptor] for this [AverageColor]. * @@ -85,7 +66,7 @@ class AverageColor : Analyser { override fun newExtractor(name: String, input: Operator, context: IndexContext): Extractor = AverageColorExtractor(input, this, null) /** - * Generates and returns a new [AverageColorRetriever] instance for this [AverageColor]. + * Generates and returns a new [DenseRetriever] instance for this [AverageColor]. * * @param field The [Schema.Field] to create an [Retriever] for. * @param query The [Query] to use with the [Retriever]. @@ -93,15 +74,14 @@ class AverageColor : Analyser { * * @return A new [Retriever] instance for this [Analyser] */ - override fun newRetrieverForQuery(field: Schema.Field, query: Query, context: QueryContext): AverageColorRetriever { - require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } + override fun newRetrieverForQuery(field: Schema.Field, query: Query, context: QueryContext): DenseRetriever { require(query is ProximityQuery<*> && query.value is Value.FloatVector) { "The query is not a ProximityQuery." } @Suppress("UNCHECKED_CAST") - return AverageColorRetriever(field, query as ProximityQuery) + return DenseRetriever(field, query as ProximityQuery, context, LinearCorrespondence(3f)) } /** - * Generates and returns a new [AverageColorRetriever] instance for this [AverageColor]. + * Generates and returns a new [DenseRetriever] instance for this [AverageColor]. * * Invoking this method involves converting the provided [FloatVectorDescriptor] into a [ProximityQuery] that can be used to retrieve similar [ImageContent] elements. * @@ -109,9 +89,7 @@ class AverageColor : Analyser { * @param descriptors An array of [FloatVectorDescriptor] elements to use with the [Retriever] * @param context The [QueryContext] to use with the [Retriever] */ - override fun newRetrieverForDescriptors(field: Schema.Field, descriptors: Collection, context: QueryContext): AverageColorRetriever { - require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } - + override fun newRetrieverForDescriptors(field: Schema.Field, descriptors: Collection, context: QueryContext): DenseRetriever { /* Prepare query parameters. */ val k = context.getProperty(field.fieldName, "limit")?.toLongOrNull() ?: 1000L val fetchVector = context.getProperty(field.fieldName, "returnDescriptor")?.toBooleanStrictOrNull() ?: false @@ -121,7 +99,7 @@ class AverageColor : Analyser { } /** - * Generates and returns a new [AverageColorRetriever] instance for this [AverageColor]. + * Generates and returns a new [DenseRetriever] instance for this [AverageColor]. * * Invoking this method involves converting the provided [ImageContent] and the [QueryContext] into a [FloatVectorDescriptor] * that can be used to retrieve similar [ImageContent] elements. @@ -130,6 +108,27 @@ class AverageColor : Analyser { * @param content An array of [Content] elements to use with the [Retriever] * @param context The [QueryContext] to use with the [Retriever] */ - override fun newRetrieverForContent(field: Schema.Field, content: Collection, context: QueryContext): AverageColorRetriever = - this.newRetrieverForDescriptors(field, analyse(content), context) + override fun newRetrieverForContent(field: Schema.Field, content: Collection, context: QueryContext): DenseRetriever = + this.newRetrieverForDescriptors(field, content.map { this.analyse(it) }, context) + + /** + * Performs the [AverageColor] analysis on the provided [List] of [ImageContent] elements. + * + * @param content The [List] of [ImageContent] elements. + * @return [List] of [FloatVectorDescriptor]s. + */ + fun analyse(content: ImageContent): FloatVectorDescriptor { + val color = floatArrayOf(0f, 0f, 0f) + val rgb = content.content.getRGBArray() + for (c in rgb) { + val container = RGBColorContainer(c) + color[0] += container.red + color[1] += container.green + color[2] += container.blue + } + + /* Generate descriptor. */ + val averageColor = RGBColorContainer(color[0] / rgb.size, color[1] / rgb.size, color[2] / rgb.size) + return FloatVectorDescriptor(UUID.randomUUID(), null, averageColor.toVector()) + } } diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColorExtractor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColorExtractor.kt index 3ecf15dff..41ffcd315 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColorExtractor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColorExtractor.kt @@ -39,6 +39,6 @@ class AverageColorExtractor(input: Operator, analyser: AverageColor */ override fun extract(retrievable: Retrievable): List { val content = retrievable.content.filterIsInstance() - return AverageColor.analyse(content).map { it.copy(retrievableId = retrievable.id, field = this@AverageColorExtractor.field) } + return content.map { (this.analyser as AverageColor).analyse(it).copy(retrievableId = retrievable.id, field = this.field) } } } \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColorRetriever.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColorRetriever.kt deleted file mode 100644 index 48a788b23..000000000 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColorRetriever.kt +++ /dev/null @@ -1,48 +0,0 @@ -package org.vitrivr.engine.core.features.averagecolor - -import io.github.oshai.kotlinlogging.KLogger -import io.github.oshai.kotlinlogging.KotlinLogging -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.flow.flow -import org.vitrivr.engine.core.model.content.element.ImageContent -import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor -import org.vitrivr.engine.core.model.metamodel.Schema -import org.vitrivr.engine.core.model.query.proximity.ProximityQuery -import org.vitrivr.engine.core.model.retrievable.Retrieved -import org.vitrivr.engine.core.model.retrievable.attributes.DistanceAttribute -import org.vitrivr.engine.core.model.retrievable.attributes.ScoreAttribute -import org.vitrivr.engine.core.model.types.Value -import org.vitrivr.engine.core.operators.retrieve.Retriever - -/** - * [Retriever] implementation for the [AverageColor] analyser. - * - * @see [AverageColor] - * - * @author Luca Rossetto - * @version 1.1.0 - */ -class AverageColorRetriever( - override val field: Schema.Field, - private val query: ProximityQuery -) : Retriever { - - private val logger: KLogger = KotlinLogging.logger {} - - companion object { - private const val MAXIMUM_DISTANCE = 3f - fun scoringFunction(retrieved: Retrieved): Float { - val distance = retrieved.filteredAttribute()?.distance ?: return 0.0f - return 1.0f - (distance / MAXIMUM_DISTANCE) - } - } - - override fun toFlow(scope: CoroutineScope) = flow { - val reader = this@AverageColorRetriever.field.getReader() - logger.debug { "Flow init with query $query" } - reader.queryAndJoin(this@AverageColorRetriever.query).forEach { - it.addAttribute(ScoreAttribute.Similarity(scoringFunction(it))) - emit(it) - } - } -} diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/dense/DenseRetriever.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/dense/DenseRetriever.kt index 94fe546cf..ea97b2da9 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/dense/DenseRetriever.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/dense/DenseRetriever.kt @@ -4,11 +4,13 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.flow.flow import org.vitrivr.engine.core.context.QueryContext import org.vitrivr.engine.core.features.AbstractRetriever +import org.vitrivr.engine.core.math.correspondence.CorrespondenceFunction import org.vitrivr.engine.core.model.content.element.ContentElement import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.query.proximity.ProximityQuery -import org.vitrivr.engine.core.util.math.ScoringFunctions +import org.vitrivr.engine.core.model.retrievable.attributes.DistanceAttribute +import org.vitrivr.engine.core.model.retrievable.attributes.ScoreAttribute /** * [DenseRetriever] implementation for proximity-based retrieval on float vector embeddings. @@ -24,10 +26,17 @@ import org.vitrivr.engine.core.util.math.ScoringFunctions * @author Fynn Faber * @version 1.0.0 */ -class DenseRetriever>(field: Schema.Field, query: ProximityQuery<*>, context: QueryContext) : AbstractRetriever(field, query, context) { +class DenseRetriever>(field: Schema.Field, query: ProximityQuery<*>, context: QueryContext, val correspondence: CorrespondenceFunction) : + AbstractRetriever(field, query, context) { override fun toFlow(scope: CoroutineScope) = flow { this@DenseRetriever.reader.queryAndJoin(this@DenseRetriever.query).forEach { - it.addAttribute(ScoringFunctions.bounded(it, 0.0f, 2.0f)) + val distance = it.filteredAttribute() + if (distance != null) { + it.addAttribute(this@DenseRetriever.correspondence(distance)) + } else { + this@DenseRetriever.logger.warn { "No distance attribute found for descriptor ${it.id}." } + it.addAttribute(ScoreAttribute.Similarity(0.0f)) + } emit(it) } } diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/math/correspondence/BoundedCorrespondence.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/math/correspondence/BoundedCorrespondence.kt new file mode 100644 index 000000000..a55644767 --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/math/correspondence/BoundedCorrespondence.kt @@ -0,0 +1,14 @@ +package org.vitrivr.engine.core.math.correspondence + +import org.vitrivr.engine.core.model.retrievable.attributes.DistanceAttribute +import org.vitrivr.engine.core.model.retrievable.attributes.ScoreAttribute + +/** + * A [CorrespondenceFunction] that is based on and upper and lower bound for the distance value. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class BoundedCorrespondence(private val min: Float = 0.0f, private val max: Float = 1.0f) : CorrespondenceFunction { + override fun invoke(distance: DistanceAttribute): ScoreAttribute.Similarity = ScoreAttribute.Similarity((this.max - distance.distance) / (this.max - this.min)) +} \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/math/correspondence/CorrespondenceFunction.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/math/correspondence/CorrespondenceFunction.kt new file mode 100644 index 000000000..9a4555529 --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/math/correspondence/CorrespondenceFunction.kt @@ -0,0 +1,20 @@ +package org.vitrivr.engine.core.math.correspondence + +import org.vitrivr.engine.core.model.retrievable.attributes.DistanceAttribute +import org.vitrivr.engine.core.model.retrievable.attributes.ScoreAttribute + +/** + * A [CorrespondenceFunction] is used to compute the [ScoreAttribute.Similarity] for a given [DistanceAttribute]. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +interface CorrespondenceFunction { + /** + * Computes the score for a given [DistanceAttribute]. + * + * @param distance [DistanceAttribute] for which to compute the score. + * @return [ScoreAttribute.Similarity] for the given [DistanceAttribute]. + */ + operator fun invoke(distance: DistanceAttribute): ScoreAttribute.Similarity +} \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/math/correspondence/LinearCorrespondence.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/math/correspondence/LinearCorrespondence.kt new file mode 100644 index 000000000..c95717a99 --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/math/correspondence/LinearCorrespondence.kt @@ -0,0 +1,14 @@ +package org.vitrivr.engine.core.math.correspondence + +import org.vitrivr.engine.core.model.retrievable.attributes.DistanceAttribute +import org.vitrivr.engine.core.model.retrievable.attributes.ScoreAttribute + +/** + * A linear [CorrespondenceFunction] that is based on a maximum distance. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class LinearCorrespondence(private val max: Float) : CorrespondenceFunction { + override fun invoke(distance: DistanceAttribute): ScoreAttribute.Similarity = ScoreAttribute.Similarity(1.0f - (distance.distance / this.max)) +} \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/ColorUtilities.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/ColorUtilities.kt new file mode 100644 index 000000000..a9883b506 --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/ColorUtilities.kt @@ -0,0 +1,35 @@ +package org.vitrivr.engine.core.model.color + +/** + * Utility class for color calculations. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +object ColorUtilities { + /** + * Calculates the average color of a collection of [RGBColorContainer]s. + * + * @param colors The [Collection] of [RGBColorContainer]s for which the average should be calculated. + * @return Average [RGBColorContainer]. + */ + fun avg(colors: Collection): RGBColorContainer { + var r = 0.0f + var g = 0.0f + var b = 0.0f + var alpha = 0.0f + var len = 0.0f + for (color in colors) { + alpha = color.alpha + r += color.red * color.alpha + g += color.green * color.alpha + b += color.blue * color.alpha + len += color.alpha + } + return if (len < 1f) { + RGBColorContainer(1.0f, 1.0f, 1.0f) + } else { + RGBColorContainer(r / len, g / len, b / len, alpha / len) + } + } +} \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/HSVColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/HSVColorContainer.kt new file mode 100644 index 000000000..080735fd4 --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/HSVColorContainer.kt @@ -0,0 +1,56 @@ +package org.vitrivr.engine.core.model.color + +import org.vitrivr.engine.core.model.types.Value +import java.awt.color.ColorSpace + +/** + * A container for HSV colors. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +@JvmInline +value class HSVColorContainer constructor(private val hsv: FloatArray) { + + init { + require(this.hsv.size == 3) { "HSVFloatColorContainer must have exactly 3 elements." } + require(this.hsv[0] in 0.0f..1.0f) { "HSVFloatColorContainer components must be between 0.0 and 1.0." } + require(this.hsv[1] in 0.0f..1.0f) { "HSVFloatColorContainer components must be between 0.0 and 1.0." } + require(this.hsv[2] in 0.0f..1.0f) { "HSVFloatColorContainer components must be between 0.0 and 1.0." } + } + + constructor(hue: Float, saturation: Float, value: Float) : this(floatArrayOf(hue, saturation, value)) + constructor(hue: Double, saturation: Double, value: Double) : this(floatArrayOf(hue.toFloat(), saturation.toFloat(), value.toFloat())) + + /** Accessor for the hue component of the [HSVColorContainer]. */ + val hue: Float + get() = this.hsv[0] + + /** Accessor for the saturation component of the [HSVColorContainer]. */ + val saturation: Float + get() = this.hsv[1] + + /** Accessor for the value component of the [HSVColorContainer]. */ + val value: Float + get() = this.hsv[2] + + /** + * Converts this [HSVColorContainer] a [Value.FloatVector] + * + * @return [Value.FloatVector] + */ + fun toVector() = Value.FloatVector(this.hsv) + + /** + * Converts this [HSVColorContainer] to a [RGBColorContainer]. + * + * @return [RGBColorContainer] + */ + fun toRGB(): RGBColorContainer { + val space = ColorSpace.getInstance(ColorSpace.TYPE_HSV) + val hsv = space.toRGB(this.hsv) + return RGBColorContainer(hsv[0], hsv[1], hsv[2]) + } + + override fun toString(): String = "HSVFloatColorContainer(H=$hue, S=$saturation, V=$value)" +} \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/LabColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/LabColorContainer.kt new file mode 100644 index 000000000..d8027b85c --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/LabColorContainer.kt @@ -0,0 +1,52 @@ +package org.vitrivr.engine.core.model.color + +import org.vitrivr.engine.core.model.types.Value +import java.awt.color.ColorSpace + +/** + * A container for LAB colors. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +@JvmInline +value class LabColorContainer constructor(private val lab: FloatArray) { + init { + require(this.lab.size == 3) { "LabColorContainer must have exactly 3 elements." } + } + + constructor(l: Float, a: Float, b: Float) : this(floatArrayOf(l, a, b)) + constructor(l: Double, a: Double, b: Double) : this(floatArrayOf(l.toFloat(), a.toFloat(), b.toFloat())) + + /** Accessor for the L component of the [LabColorContainer]. */ + val l: Float + get() = this.lab[0] + + /** Accessor for the A component of the [LabColorContainer]. */ + val a: Float + get() = this.lab[1] + + /** Accessor for the B component of the [LabColorContainer]. */ + val b: Float + get() = this.lab[2] + + /** + * Converts this [LabColorContainer] a [Value.FloatVector] + * + * @return [Value.FloatVector] + */ + fun toVector() = Value.FloatVector(this.lab) + + /** + * Converts this [XYZColorContainer] to a [RGBColorContainer]. + * + * @return [RGBColorContainer] + */ + fun toRGB(): RGBColorContainer { + val space = ColorSpace.getInstance(ColorSpace.TYPE_Lab) + val lab = space.toRGB(this.lab) + return RGBColorContainer(lab[0], lab[1], lab[2]) + } + + override fun toString(): String = "LabFloatColorContainer(L=$l, A=$a, B=$b)" +} \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/MutableRGBFloatColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/MutableRGBFloatColorContainer.kt deleted file mode 100644 index 26a2f3721..000000000 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/MutableRGBFloatColorContainer.kt +++ /dev/null @@ -1,105 +0,0 @@ -package org.vitrivr.engine.core.model.color - -import kotlin.math.max -import kotlin.math.min -import kotlin.math.sqrt - -data class MutableRGBFloatColorContainer(override var red: Float, override var green: Float, override var blue: Float) : - RGBFloatColorContainer(red, green, blue) { - - constructor() : this(0f, 0f, 0f) - - override operator fun plus(other: RGBFloatColorContainer): MutableRGBFloatColorContainer = - MutableRGBFloatColorContainer(this.red + other.red, this.green + other.green, this.blue + other.blue) - - override operator fun plus(other: RGBByteColorContainer): MutableRGBFloatColorContainer = - MutableRGBFloatColorContainer( - this.red + (other.red.toFloat() / 255f), - this.green + (other.green.toFloat() / 255f), - this.blue + (other.blue.toFloat() / 255f) - ) - - operator fun plusAssign(other: RGBFloatColorContainer) { - this.red += other.red - this.green += other.green - this.blue += other.blue - } - - operator fun plusAssign(other: RGBByteColorContainer) { - this.red += other.red.toFloat() / 255f - this.green += other.green.toFloat() / 255f - this.blue += other.blue.toFloat() / 255f - } - - override operator fun minus(other: RGBFloatColorContainer): MutableRGBFloatColorContainer = - MutableRGBFloatColorContainer(this.red - other.red, this.green - other.green, this.blue - other.blue) - - override operator fun minus(other: RGBByteColorContainer): MutableRGBFloatColorContainer = - MutableRGBFloatColorContainer( - this.red - (other.red.toFloat() / 255f), - this.green - (other.green.toFloat() / 255f), - this.blue - (other.blue.toFloat() / 255f) - ) - - operator fun times(value: Float) : MutableRGBFloatColorContainer = MutableRGBFloatColorContainer(this.red * value, this.green * value, this.blue * value) - - operator fun timesAssign(value: Float) { - this.red *= value - this.green *= value - this.blue *= value - } - - operator fun div(value: Float) : MutableRGBFloatColorContainer = MutableRGBFloatColorContainer(this.red / value, this.green / value, this.blue / value) - - operator fun divAssign(value: Float) { - this.red /= value - this.green /= value - this.blue /= value - } - - /** - * Clamps the values of each color channel to [0f, 1f] - */ - override fun clamped(): MutableRGBFloatColorContainer = - MutableRGBFloatColorContainer( - min(1f, max(0f, this.red)), - min(1f, max(0f, this.green)), - min(1f, max(0f, this.blue)) - ) - - /** - * Clamps the values of each color channel to [0f, 1f] in this instance - */ - fun clamp(): MutableRGBFloatColorContainer { - this.red = min(1f, max(0f, this.red)) - this.green = min(1f, max(0f, this.green)) - this.blue = min(1f, max(0f, this.blue)) - return this - } - - /** - * Normalizes the magnitude of the color vector to 1 - */ - override fun normalized(): MutableRGBFloatColorContainer { - val magnitude = sqrt(this.red * this.red + this.green * this.green + this.blue * this.blue) - return if (magnitude > 0f) { - MutableRGBFloatColorContainer(this.red / magnitude, this.green / magnitude, this.blue / magnitude) - } else { - this - } - } - - /** - * Normalizes the magnitude of this color vector to 1 - */ - fun normalize(): MutableRGBFloatColorContainer { - val magnitude = sqrt(this.red * this.red + this.green * this.green + this.blue * this.blue) - if (magnitude > 0f) { - this.red /= magnitude - this.green /= magnitude - this.blue /= magnitude - } - return this - } - -} diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBByteColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBByteColorContainer.kt deleted file mode 100644 index 8db668c17..000000000 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBByteColorContainer.kt +++ /dev/null @@ -1,45 +0,0 @@ -package org.vitrivr.engine.core.model.color - -/** - * A container for RGB colors. - * - * @version 1.1.0 - * @author Luca Rossetto - * @author Ralph Gasser - */ -@JvmInline -value class RGBByteColorContainer(private val rgb: Int) { - /** - * Constructor for creating a [RGBByteColorContainer] from RGB [UByte] values. - * - * @param red Red component. - * @param green Green component. - * @param blue Blue component. - */ - constructor(red: UByte, green: UByte, blue: UByte) : this(blue.toInt() and 0xFF or ((green.toInt() and 0xFF) shl 8) or ((red.toInt() and 0xFF) shl 16)) - - /** Red component of this [RGBByteColorContainer]. */ - val red: UByte - get() = (this.rgb shr 16 and 0xFF).toUByte() - - /** Green component of this [RGBByteColorContainer]. */ - val green: UByte - get() = (this.rgb shr 8 and 0xFF).toUByte() - - /** Blue component of this [RGBByteColorContainer]. */ - val blue: UByte - get() = (this.rgb and 0xFF).toUByte() - - /** - * - */ - fun toFloatContainer(): RGBFloatColorContainer = - RGBFloatColorContainer(this.red.toFloat() / 255f, this.green.toFloat() / 255f, this.blue.toFloat() / 255f) - - /** - * Converts this [RGBByteColorContainer] to an [Int] representation. - * - * @return [Int] representation of this [RGBByteColorContainer]. - */ - fun toRGBInt(): Int = this.rgb -} diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBColorContainer.kt new file mode 100644 index 000000000..498a6fad9 --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBColorContainer.kt @@ -0,0 +1,186 @@ +package org.vitrivr.engine.core.model.color + +import org.vitrivr.engine.core.model.types.Value +import java.awt.color.ColorSpace +import kotlin.math.max +import kotlin.math.min +import kotlin.math.sqrt + +/** + * A container for RGB colors. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +@JvmInline +value class RGBColorContainer constructor(private val rgb: FloatArray) { + + init { + require(this.rgb.size == 4) { "RGBFloatColorContainer must have exactly 4 elements." } + require(this.rgb[0] in 0.0f..1.0f) { "RGBFloatColorContainer components must be between 0.0 and 1.0." } + require(this.rgb[1] in 0.0f..1.0f) { "RGBFloatColorContainer components must be between 0.0 and 1.0." } + require(this.rgb[2] in 0.0f..1.0f) { "RGBFloatColorContainer components must be between 0.0 and 1.0." } + require(this.rgb[3] in 0.0f..1.0f) { "RGBFloatColorContainer components must be between 0.0 and 1.0." } + } + + constructor(rgb: Int) : this((rgb shr 16 and 0xFF).toByte(), (rgb shr 8 and 0xFF).toByte(), (rgb and 0xFF).toByte()) + constructor(red: Byte, green: Byte, blue: Byte, alpha: Byte = Byte.MAX_VALUE) : this(red.toFloat() / 255f, green.toFloat() / 255f, blue.toFloat() / 255f, alpha.toFloat() / 255f) + constructor(red: Int, green: Int, blue: Int, alpha: Int = 255) : this(red.toFloat() / 255f, green.toFloat() / 255f, blue.toFloat() / 255f, alpha.toFloat() / 255f) + constructor(r: Double, g: Double, b: Double, alpha: Double = 1.0) : this(r.toFloat(), g.toFloat(), b.toFloat(), alpha.toFloat()) + constructor(red: Float, green: Float, blue: Float, alpha: Float = 1.0f) : this(floatArrayOf(red, green, blue, alpha)) + + /** Accessor for the red component of the [RGBColorContainer]. */ + val red: Float + get() = this.rgb[0] + + /** Accessor for the green component of the [RGBColorContainer]. */ + val green: Float + get() = this.rgb[1] + + /** Accessor for the blue component of the [RGBColorContainer]. */ + val blue: Float + get() = this.rgb[2] + + /** Accessor for the alpha component of the [RGBColorContainer]. */ + val alpha: Float + get() = this.rgb[3] + + /** + * Converts the [red] component of this [RGBColorContainer] to an [Int] representation. + * + * @return [Int] representation of [blue] color + */ + fun redAsInt(): Int = (this.red * 255).toInt() + + /** + * Converts the [green] component of this [RGBColorContainer] to an [Int] representation. + * + * @return [Int] representation of [blue] color + */ + fun greenAsInt(): Int = (this.green * 255).toInt() + + /** + * Converts the [blue] component of this [RGBColorContainer] to an [Int] representation. + * + * @return [Int] representation of [blue] color + */ + fun blueAsInt(): Int = (this.blue * 255).toInt() + + /** + * Converts the [alpha] component of this [RGBColorContainer] to an [Int] representation. + * + * @return [Int] representation of [alpha] color + */ + fun alphaAsInt(): Int = (this.alpha * 255).toInt() + + /** + * Adds this [RGBColorContainer] to another [RGBColorContainer]. + * + * @param other [RGBColorContainer] to add. + * @return [RGBColorContainer] + */ + operator fun plus(other: RGBColorContainer): RGBColorContainer = + RGBColorContainer(this.red + other.red, this.green + other.green, this.blue + other.blue) + + /** + * Subtracts a [RGBColorContainer] from this [RGBColorContainer]. + * + * @param other [RGBColorContainer] to subtract. + * @return [RGBColorContainer] + */ + operator fun minus(other: RGBColorContainer): RGBColorContainer = RGBColorContainer( + this.red - other.red, + this.green - other.green, + this.blue - other.blue + ) + + /** + * Clamps the values of each color channel to [0f, 1f] + */ + fun clamped(): RGBColorContainer = RGBColorContainer(min(1f, max(0f, this.red)), min(1f, max(0f, this.green)), min(1f, max(0f, this.blue))) + + /** + * Normalizes the magnitude of the color vector to 1 + */ + fun normalized(): RGBColorContainer { + val magnitude = sqrt(this.red * this.red + this.green * this.green + this.blue * this.blue) + return if (magnitude > 0f) { + RGBColorContainer(this.red / magnitude, this.green / magnitude, this.blue / magnitude) + } else { + this + } + } + + /** + * Converts this [RGBColorContainer] to an [Int] representation. + * + * @return [Int] representation of RGB color + */ + fun toRGBInt(): Int = this.blueAsInt() and 0XFF or ((this.greenAsInt() and 0xFF) shl 8) or ((this.redAsInt() and 0xFF) shl 16) + + /** + * Converts this [RGBColorContainer] to a [HSVColorContainer]. + * + * @return [HSVColorContainer] + */ + fun toHSV(): HSVColorContainer { + val space = ColorSpace.getInstance(ColorSpace.TYPE_HSV) + return HSVColorContainer(space.fromRGB(this.rgb)) + } + + /** + * Converts this [RGBColorContainer] to a [LabColorContainer]. + * + * @return [LabColorContainer] + */ + fun toLab(): LabColorContainer { + val space = ColorSpace.getInstance(ColorSpace.TYPE_Lab) + return LabColorContainer(space.fromRGB(this.rgb)) + } + + /** + * Converts this [RGBColorContainer] to a [XYZColorContainer]. + * + * @return [XYZColorContainer] + */ + fun toXYZ(): XYZColorContainer { + val space = ColorSpace.getInstance(ColorSpace.TYPE_XYZ) + return XYZColorContainer(space.fromRGB(this.rgb)) + } + + /** + * Converts this [RGBColorContainer] to a [YCbCrColorContainer]. + * + * @return [YCbCrColorContainer] + */ + fun toYCbCr(): YCbCrColorContainer { + val space = ColorSpace.getInstance(ColorSpace.TYPE_YCbCr) + return YCbCrColorContainer(space.fromRGB(this.rgb)) + } + + /** + * Converts this [RGBColorContainer] as a [List] of [Float]s. + * + * @return [List] of [Float]s + */ + fun toList() = listOf(this.red, this.green, this.blue) + + /** + * Converts this [RGBColorContainer] a [Value.FloatVector] + * + * @return [Value.FloatVector] + */ + fun toVector() = Value.FloatVector(this.rgb) + + /** + * Calculates distance to another [RGBColorContainer]. + * + * @param other [RGBColorContainer] to calculate distance to. + * @return Distance to [other]. + */ + fun distanceTo(other: RGBColorContainer): Float = sqrt( + (this.red - other.red) * (this.red - other.red) + (this.green - other.green) * (this.green - other.green) + (this.blue - other.blue) * (this.blue - other.blue) + ) + + override fun toString(): String = "RGBFloatColorContainer(R=$red, G=$green, B=$blue)" +} diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBFloatColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBFloatColorContainer.kt deleted file mode 100644 index 3218bc69e..000000000 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBFloatColorContainer.kt +++ /dev/null @@ -1,99 +0,0 @@ -package org.vitrivr.engine.core.model.color - -import org.vitrivr.engine.core.model.types.Value -import kotlin.math.max -import kotlin.math.min -import kotlin.math.roundToInt -import kotlin.math.sqrt - -open class RGBFloatColorContainer(open val red: Float, open val green: Float, open val blue: Float) { - - constructor(floats: List) : this(floats[0], floats[1], floats[2]) - - open operator fun plus(other: RGBFloatColorContainer): RGBFloatColorContainer = - RGBFloatColorContainer(this.red + other.red, this.green + other.green, this.blue + other.blue) - - open operator fun plus(other: RGBByteColorContainer): RGBFloatColorContainer = RGBFloatColorContainer( - this.red + (other.red.toFloat() / 255f), - this.green + (other.green.toFloat() / 255f), - this.blue + (other.blue.toFloat() / 255f) - ) - - open operator fun minus(other: RGBFloatColorContainer): RGBFloatColorContainer = - RGBFloatColorContainer(this.red - other.red, this.green - other.green, this.blue - other.blue) - - open operator fun minus(other: RGBByteColorContainer): RGBFloatColorContainer = - RGBFloatColorContainer( - this.red - (other.red.toFloat() / 255f), - this.green - (other.green.toFloat() / 255f), - this.blue - (other.blue.toFloat() / 255f) - ) - - /** - * Clamps the values of each color channel to [0f, 1f] - */ - open fun clamped(): RGBFloatColorContainer = - RGBFloatColorContainer(min(1f, max(0f, this.red)), min(1f, max(0f, this.green)), min(1f, max(0f, this.blue))) - - /** - * Normalizes the magnitude of the color vector to 1 - */ - open fun normalized(): RGBFloatColorContainer { - val magnitude = sqrt(this.red * this.red + this.green * this.green + this.blue * this.blue) - return if (magnitude > 0f) { - RGBFloatColorContainer(this.red / magnitude, this.green / magnitude, this.blue / magnitude) - } else { - this - } - } - - fun toMutable(): MutableRGBFloatColorContainer = MutableRGBFloatColorContainer(this.red, this.green, this.blue) - - fun toByteContainer(): RGBByteColorContainer = RGBByteColorContainer( - (min(1f, max(0f, this.red)) * 255f).roundToInt().toUByte(), - (min(1f, max(0f, this.green)) * 255f).roundToInt().toUByte(), - (min(1f, max(0f, this.blue)) * 255f).roundToInt().toUByte() - ) - - /** - * Converts this [RGBFloatColorContainer] as a [List] of [Float]s. - * - * @return [List] of [Float]s - */ - fun toList() = listOf(this.red, this.green, this.blue) - - /** - * Converts this [RGBFloatColorContainer] as a [List] of [Value.Float]s. - * - * @return [List] of [Value.Float]s - */ - fun toVector() = Value.FloatVector(floatArrayOf(this.red, this.green, this.blue)) - - fun distanceTo(other: RGBFloatColorContainer): Float = sqrt( - (this.red - other.red) * (this.red - other.red) + (this.green - other.green) * (this.green - other.green) + (this.blue - other.blue) * (this.blue - other.blue) - ) - - override fun equals(other: Any?): Boolean { - if (this === other) return true - if (other !is RGBFloatColorContainer) return false - - if (red != other.red) return false - if (green != other.green) return false - if (blue != other.blue) return false - - return true - } - - override fun hashCode(): Int { - var result = red.hashCode() - result = 31 * result + green.hashCode() - result = 31 * result + blue.hashCode() - return result - } - - override fun toString(): String { - return "RGBFloatColorContainer(red=$red, green=$green, blue=$blue)" - } - - -} diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/XYZColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/XYZColorContainer.kt new file mode 100644 index 000000000..f6b5b0cd6 --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/XYZColorContainer.kt @@ -0,0 +1,51 @@ +package org.vitrivr.engine.core.model.color + +import org.vitrivr.engine.core.model.types.Value +import java.awt.color.ColorSpace + +/** + * A container for XYZ colors. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +@JvmInline +value class XYZColorContainer constructor(private val xyz: FloatArray) { + + constructor(x: Float, y: Float, z: Float) : this(floatArrayOf(x, y, z)) + constructor(x: Double, y: Double, z: Double) : this(floatArrayOf(x.toFloat(), y.toFloat(), z.toFloat())) + + /** Accessor for the L component of the [LabColorContainer]. */ + val x: Float + get() = this.xyz[0] + + /** Accessor for the A component of the [LabColorContainer]. */ + val y: Float + get() = this.xyz[1] + + /** Accessor for the B component of the [LabColorContainer]. */ + val z: Float + get() = this.xyz[2] + + /** + * Converts this [XYZColorContainer] to a [RGBColorContainer]. + * + * @return [RGBColorContainer] + */ + fun toRGB(): RGBColorContainer { + val space = ColorSpace.getInstance(ColorSpace.TYPE_XYZ) + val xyz = space.toRGB(this.xyz) + return RGBColorContainer(xyz[0], xyz[1], xyz[2]) + } + + /** + * Converts this [XYZColorContainer] a [Value.FloatVector] + * + * @return [Value.FloatVector] + */ + fun toVector() = Value.FloatVector(this.xyz) + + override fun toString(): String { + return "XYZFloatColorContainer(X=$x, Y=$y, Z=$z)" + } +} \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/YCbCrColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/YCbCrColorContainer.kt new file mode 100644 index 000000000..5eef673aa --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/YCbCrColorContainer.kt @@ -0,0 +1,41 @@ +package org.vitrivr.engine.core.model.color + +import java.awt.color.ColorSpace + +/** + * A container for YCbCr colors. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +@JvmInline +value class YCbCrColorContainer constructor(private val ycbcr: FloatArray) { + init { + require(this.ycbcr.size == 3) { "YCbCrColorContainer must have exactly 3 elements." } + } + + constructor(y: Float, cb: Float, cr: Float) : this(floatArrayOf(y, cb, cr)) + + /** Accessor for the L component of the [LabColorContainer]. */ + val y: Float + get() = this.ycbcr[0] + + /** Accessor for the A component of the [LabColorContainer]. */ + val cb: Float + get() = this.ycbcr[1] + + /** Accessor for the B component of the [LabColorContainer]. */ + val cr: Float + get() = this.ycbcr[2] + + /** + * Converts this [YCbCrColorContainer] to a [RGBColorContainer]. + * + * @return [RGBColorContainer] + */ + fun toRGB(): RGBColorContainer { + val space = ColorSpace.getInstance(ColorSpace.TYPE_YCbCr) + val rgb = space.toRGB(this.ycbcr) + return RGBColorContainer(rgb[0], rgb[1], rgb[2]) + } +} \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/LabelDescriptor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/LabelDescriptor.kt index 15d42437e..0e1325545 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/LabelDescriptor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/LabelDescriptor.kt @@ -27,6 +27,14 @@ class LabelDescriptor( ) } + constructor( + id: DescriptorId, + retrievableId: RetrievableId?, + label: String, + confidence: Float = 1f, + field: Schema.Field<*, LabelDescriptor>? = null + ) : this(id, retrievableId, mapOf("label" to Value.String(label), "confidence" to Value.Float(confidence)), field) + /** The stored label. */ val label: Value.String by this.values @@ -41,5 +49,6 @@ class LabelDescriptor( * @param field [Schema.Field] the new [LabelDescriptor] belongs to. * @return Copy of this [LabelDescriptor]. */ - override fun copy(id: DescriptorId, retrievableId: RetrievableId?, field: Schema.Field<*, LabelDescriptor>?) = LabelDescriptor(id, retrievableId, HashMap(this.values), this.field) + override fun copy(id: DescriptorId, retrievableId: RetrievableId?, field: Schema.Field<*, LabelDescriptor>?) = + LabelDescriptor(id, retrievableId, HashMap(this.values), this.field) } diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/util/math/MathHelper.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/util/math/MathHelper.kt index db6e57a54..c2bd52afd 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/util/math/MathHelper.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/util/math/MathHelper.kt @@ -8,10 +8,22 @@ import kotlin.math.sqrt * A collection of helper functions for common, mathematical operations. * * @author Ralph Gasser - * @version 1.0.0 + * @version 1.0.1 */ object MathHelper { + /** Square root of 2. */ + val SQRT2: Double = sqrt(2.0) + + /** Square root of 2 as a [Float]. */ + val SQRT2_f: Float = SQRT2.toFloat() + + /** Square root of 1/2. */ + val SQRT1_2: Double = sqrt(0.5) + + /** Square root of 1/2 as a [Float]. */ + val SQRT1_2f: Float = SQRT1_2.toFloat() + /** * Normalizes a [Array] of [Value.Float] with respect to the L2 (euclidian) norm. * The method will return a new array and leave the original array unchanged. diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/util/math/ScoringFunctions.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/util/math/ScoringFunctions.kt deleted file mode 100644 index faf5a316e..000000000 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/util/math/ScoringFunctions.kt +++ /dev/null @@ -1,37 +0,0 @@ -package org.vitrivr.engine.core.util.math - -import org.vitrivr.engine.core.model.retrievable.Retrieved -import org.vitrivr.engine.core.model.retrievable.attributes.DistanceAttribute -import org.vitrivr.engine.core.model.retrievable.attributes.ScoreAttribute - -/** - * A collection of [ScoringFunctions] that can be used to score [Retrieved] object that come with a [DistanceAttribute]. - * - * @author Ralph Gasser - * @version 1.0.0 - */ -object ScoringFunctions { - - /** - * A scoring function that assumes a defined minimum and maximum and normalizes the distance between these two values. - * - * @param retrieved [Retrieved] object to score. - * @param min Minimum value. Default is 0.0. - * @param max Maximum value. Default is 1.0. - */ - fun bounded(retrieved: Retrieved, min: Float = 0.0f, max: Float = 1.0f): ScoreAttribute { - val distance = retrieved.filteredAttribute()?.distance ?: return ScoreAttribute.Similarity(0.0f) - return ScoreAttribute.Similarity((max-distance) / (max - min)) - } - - /** - * A scoring functions that assumes a defined maximum and subtracts the distance from that maximum to obtain a score. - * - * @param retrieved [Retrieved] object to score. - * @param max Maximum value. Default is 1.0. - */ - fun max(retrieved: Retrieved, max: Float = 1.0f): ScoreAttribute { - val distance = retrieved.filteredAttribute()?.distance ?: return ScoreAttribute.Unbound(0.0f) - return ScoreAttribute.Unbound(max - distance) - } -} diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/util/math/StatisticsHelper.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/util/math/StatisticsHelper.kt new file mode 100644 index 000000000..9059f1af8 --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/util/math/StatisticsHelper.kt @@ -0,0 +1,31 @@ +package org.vitrivr.engine.core.util.math + +/** + * A collection of helper methods for basic statistics. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +object StatisticsHelper { + /** + * Extracts the median value from a [IntArray] histogram. + * + * @param hist The [IntArray] representing the histogram. + * @return The median value. + */ + fun medianFromHistogram(hist: IntArray): Int { + var posL = 0 + var posR = hist.size - 1 + var sumL = hist[posL] + var sumR = hist[posR] + + while (posL < posR) { + if (sumL < sumR) { + sumL += hist[++posL] + } else { + sumR += hist[--posR] + } + } + return posL + } +} \ No newline at end of file diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/AverageImageContentAggregator.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/AverageImageContentAggregator.kt index 397cb4855..9d56851f0 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/AverageImageContentAggregator.kt +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/AverageImageContentAggregator.kt @@ -2,8 +2,7 @@ package org.vitrivr.engine.index.aggregators.image import org.vitrivr.engine.core.context.Context import org.vitrivr.engine.core.context.IndexContext -import org.vitrivr.engine.core.model.color.MutableRGBFloatColorContainer -import org.vitrivr.engine.core.model.color.RGBByteColorContainer +import org.vitrivr.engine.core.model.color.RGBColorContainer import org.vitrivr.engine.core.model.content.decorators.SourcedContent import org.vitrivr.engine.core.model.content.decorators.TemporalContent import org.vitrivr.engine.core.model.content.element.ContentElement @@ -52,17 +51,23 @@ class AverageImageContentAggregator : TransformerFactory { val firstImage = images.first() val height = firstImage.height val width = firstImage.width - val colors = List(firstImage.width * firstImage.height) { MutableRGBFloatColorContainer() } + val colors = List(firstImage.width * firstImage.height) { floatArrayOf(0f, 0f, 0f) } images.forEach { imageContent -> require(imageContent.height == height && imageContent.width == width) { "Unable to aggregate images! All images must have same dimension." } imageContent.content.getRGBArray().forEachIndexed { index, color -> - colors[index] += RGBByteColorContainer(color) + val rgb = RGBColorContainer(color) + colors[index][0] += rgb.red + colors[index][1] += rgb.green + colors[index][2] += rgb.blue } } val div = images.size.toFloat() - val intColors = colors.map { - (it / div).toByteContainer().toRGBInt() + val intColors = colors.map { c -> + c[0] /= div + c[1] /= div + c[2] /= div + RGBColorContainer(c).toRGBInt() }.toIntArray() /* Prepare buffered image. */ diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/RepresentativeImageContentAggregator.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/RepresentativeImageContentAggregator.kt index 51ddbec61..f2218e52f 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/RepresentativeImageContentAggregator.kt +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/RepresentativeImageContentAggregator.kt @@ -2,8 +2,7 @@ package org.vitrivr.engine.index.aggregators.image import org.vitrivr.engine.core.context.Context import org.vitrivr.engine.core.context.IndexContext -import org.vitrivr.engine.core.model.color.MutableRGBFloatColorContainer -import org.vitrivr.engine.core.model.color.RGBByteColorContainer +import org.vitrivr.engine.core.model.color.RGBColorContainer import org.vitrivr.engine.core.model.content.element.ContentElement import org.vitrivr.engine.core.model.content.element.ImageContent import org.vitrivr.engine.core.model.retrievable.Ingested @@ -52,23 +51,29 @@ class RepresentativeImageContentAggregator : TransformerFactory { val firstImage = images.first() val height = firstImage.height val width = firstImage.width - val colors = List(firstImage.width * firstImage.height) { MutableRGBFloatColorContainer() } + val colors = List(firstImage.width * firstImage.height) { floatArrayOf(0.0f, 0.0f, 0.0f) } images.forEach { imageContent -> require(imageContent.height == height && imageContent.width == width) { "Unable to aggregate images! All images must have same dimension." } imageContent.content.getRGBArray().forEachIndexed { index, color -> - colors[index] += RGBByteColorContainer(color) + val rgb = RGBColorContainer(color) + colors[index][0] += rgb.red + colors[index][1] += rgb.green + colors[index][2] += rgb.blue } } /* normalize */ val div = images.size.toFloat() - colors.forEach { it /= div } + for (color in colors) { + color[0] /= div + color[1] /= div + color[2] /= div + } /* find image with smallest pixel-wise distance */ val mostRepresentative = images.minBy { imageContent -> - imageContent.content.getRGBArray().mapIndexed { index, color -> - RGBByteColorContainer(color).toFloatContainer().distanceTo(colors[index]) + RGBColorContainer(color).distanceTo(RGBColorContainer(colors[index])) }.sum() } diff --git a/vitrivr-engine-module-features/build.gradle b/vitrivr-engine-module-features/build.gradle index e693c8bdb..f1a9e10db 100644 --- a/vitrivr-engine-module-features/build.gradle +++ b/vitrivr-engine-module-features/build.gradle @@ -9,6 +9,9 @@ repositories { dependencies { api project(':vitrivr-engine-core') + + /* BoofCV for image modification and processing. */ + implementation group: 'org.boofcv', name: 'boofcv-io', version: version_boofcv } /* Publication of vitrivr engine query to Maven Central. */ diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/cld/CLD.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/cld/CLD.kt new file mode 100644 index 000000000..da9d6ca86 --- /dev/null +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/cld/CLD.kt @@ -0,0 +1,232 @@ +package org.vitrivr.engine.module.features.feature.cld + +import org.vitrivr.engine.core.context.IndexContext +import org.vitrivr.engine.core.context.QueryContext +import org.vitrivr.engine.core.features.dense.DenseRetriever +import org.vitrivr.engine.core.math.correspondence.LinearCorrespondence +import org.vitrivr.engine.core.model.color.ColorUtilities +import org.vitrivr.engine.core.model.color.RGBColorContainer +import org.vitrivr.engine.core.model.content.Content +import org.vitrivr.engine.core.model.content.element.ContentElement +import org.vitrivr.engine.core.model.content.element.ImageContent +import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor +import org.vitrivr.engine.core.model.metamodel.Analyser +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.query.Query +import org.vitrivr.engine.core.model.query.proximity.ProximityQuery +import org.vitrivr.engine.core.model.retrievable.Retrievable +import org.vitrivr.engine.core.model.types.Value +import org.vitrivr.engine.core.operators.Operator +import org.vitrivr.engine.core.operators.ingest.Extractor +import org.vitrivr.engine.core.operators.retrieve.Retriever +import org.vitrivr.engine.core.util.extension.getRGBArray +import org.vitrivr.engine.core.util.math.MathHelper.SQRT1_2 +import java.util.* +import kotlin.math.cos +import kotlin.math.floor +import kotlin.reflect.KClass + +/** + * An MPEG 7 Color Layout (CLD) [Analyser] for [ImageContent] objects. + * + * Migrated from Cineast. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class CLD : Analyser { + companion object { + private const val VECTOR_SIZE = 12 + + + private val SCAN: IntArray = intArrayOf( + 0, 1, 8, 16, 9, 2, 3, 10, 17, 24, 32, + 25, 18, 11, 4, 5, 12, 19, 26, 33, 40, 48, 41, 34, 27, 20, 13, 6, 7, + 14, 21, 28, 35, 42, 49, 56, 57, 50, 43, 36, 29, 22, 15, 23, 30, 37, + 44, 51, 58, 59, 52, 45, 38, 31, 39, 46, 53, 60, 61, 54, 47, 55, 62, + 63 + ) + } + + override val contentClasses: Set>> = setOf(ImageContent::class) + override val descriptorClass: KClass = FloatVectorDescriptor::class + + /** + * Generates a prototypical [FloatVectorDescriptor] for this [CLD]. + * + * @param field [Schema.Field] to create the prototype for. + * @return [FloatVectorDescriptor] + */ + override fun prototype(field: Schema.Field<*, *>): FloatVectorDescriptor = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.FloatVector(VECTOR_SIZE)) + + /** + * Generates and returns a new [CLDExtractor] instance for this [CLD]. + * + * @param name The name of the [CLDExtractor]. + * @param input The [Operator] that acts as input to the new [Extractor]. + * @param context The [IndexContext] to use with the [Extractor]. + * + * @return A new [Extractor] instance for this [Analyser] + * @throws [UnsupportedOperationException], if this [Analyser] does not support the creation of an [Extractor] instance. + */ + override fun newExtractor(name: String, input: Operator, context: IndexContext) = CLDExtractor(input, this, null) + + /** + * Generates and returns a new [CLDExtractor] instance for this [CLD]. + * + * @param field The [Schema.Field] to create an [Extractor] for. + * @param input The [Operator] that acts as input to the new [Extractor]. + * @param context The [IndexContext] to use with the [Extractor]. + * + * @return A new [Extractor] instance for this [Analyser] + * @throws [UnsupportedOperationException], if this [Analyser] does not support the creation of an [Extractor] instance. + */ + override fun newExtractor(field: Schema.Field, input: Operator, context: IndexContext) = CLDExtractor(input, this, field) + + /** + * Generates and returns a new [DenseRetriever] instance for this [CLD]. + * + * @param field The [Schema.Field] to create an [Retriever] for. + * @param query The [Query] to use with the [Retriever]. + * @param context The [QueryContext] to use with the [Retriever]. + * + * @return A new [DenseRetriever] instance for this [CLD] + */ + override fun newRetrieverForQuery(field: Schema.Field, query: Query, context: QueryContext): DenseRetriever { + require(query is ProximityQuery<*> && query.value is Value.FloatVector) { "The query is not a ProximityQuery." } + @Suppress("UNCHECKED_CAST") + return DenseRetriever(field, query as ProximityQuery, context, LinearCorrespondence(490f)) + } + + /** + * Generates and returns a new [DenseRetriever] instance for this [CLD]. + * + * Invoking this method involves converting the provided [FloatVectorDescriptor] into a [ProximityQuery] that can be used to retrieve similar [ImageContent] elements. + * + * @param field The [Schema.Field] to create an [Retriever] for. + * @param descriptors An array of [FloatVectorDescriptor] elements to use with the [Retriever] + * @param context The [QueryContext] to use with the [Retriever] + */ + override fun newRetrieverForDescriptors(field: Schema.Field, descriptors: Collection, context: QueryContext): DenseRetriever { + /* Prepare query parameters. */ + val k = context.getProperty(field.fieldName, "limit")?.toLongOrNull() ?: 1000L + val fetchVector = context.getProperty(field.fieldName, "returnDescriptor")?.toBooleanStrictOrNull() ?: false + + /* Return retriever. */ + return this.newRetrieverForQuery(field, ProximityQuery(value = descriptors.first().vector, k = k, fetchVector = fetchVector), context) + } + + /** + * Generates and returns a new [DenseRetriever] instance for this [CLD]. + * + * Invoking this method involves converting the provided [ImageContent] and the [QueryContext] into a [FloatVectorDescriptor] + * that can be used to retrieve similar [ImageContent] elements. + * + * @param field The [Schema.Field] to create an [Retriever] for. + * @param content An array of [Content] elements to use with the [Retriever] + * @param context The [QueryContext] to use with the [Retriever] + */ + override fun newRetrieverForContent(field: Schema.Field, content: Collection, context: QueryContext) = this.newRetrieverForDescriptors(field, content.map { this.analyse(it) }, context) + + /** + * Performs the [CLD] analysis on the provided [ImageContent] and returns a [FloatVectorDescriptor] that represents the result. + * + * @param content [ImageContent] to be analysed. + * @return [FloatVectorDescriptor] result of the analysis. + */ + fun analyse(content: ImageContent): FloatVectorDescriptor { + val tmpList: ArrayList = ArrayList(content.content.width * content.content.height) + + /* Extract and normalize colors based on alpha channel. */ + val colors: IntArray = content.content.getRGBArray() + for (c in colors) { + val rgb = RGBColorContainer(c) + if (rgb.alpha < 127) { + tmpList.add(255) + } else { + tmpList.add(c) + } + } + + /* Create partitions. */ + val partitions: ArrayList> = partition(tmpList, content.content.width, content.content.height) + val rgbs = IntArray(64) + for (i in partitions.indices) { + rgbs[i] = ColorUtilities.avg(partitions[i].map { RGBColorContainer(it) }).toRGBInt() + } + + /* Obtain YCbCr values and calculate DCT. */ + val ycbcrs = Array(3) { FloatArray(64) } + for (i in 0..63) { + val c = RGBColorContainer(rgbs[i]).toYCbCr() + ycbcrs[0][i] = c.y + ycbcrs[1][i] = c.cb + ycbcrs[2][i] = c.cr + } + ycbcrs[0] = this.dct(ycbcrs[0]) + ycbcrs[1] = this.dct(ycbcrs[1]) + ycbcrs[2] = this.dct(ycbcrs[2]) + + /* Obtain CLD. */ + val cld = Value.FloatVector( + floatArrayOf( + ycbcrs[0][0].toFloat(), ycbcrs[0][1].toFloat(), ycbcrs[0][2].toFloat(), ycbcrs[0][3].toFloat(), ycbcrs[0][4].toFloat(), ycbcrs[0][5].toFloat(), + ycbcrs[1][0].toFloat(), ycbcrs[1][1].toFloat(), ycbcrs[1][2].toFloat(), + ycbcrs[2][0].toFloat(), ycbcrs[2][1].toFloat(), ycbcrs[2][2].toFloat() + ) + ) + return FloatVectorDescriptor(vector = cld) + } + + /** + * Calculates the Discrete Cosine Transform (DCT) of a given block. + * + * Based on c implementation by Berk ATABEK (http://www.batabek.com/) + * + * @param block Block for which to calculate the DCT. + * @return The DCT of the block. + */ + private fun dct(block: FloatArray): FloatArray { + var sum: Double + var cu: Double + var cv: Double + val temp = FloatArray(64) + + for (u in 0..7) { + for (v in 0..7) { + sum = 0.0 + cu = if ((u == 0)) SQRT1_2 else 1.0 + cv = if ((v == 0)) SQRT1_2 else 1.0 + for (x in 0..7) { + for (y in 0..7) { + sum += (block[x * 8 + y] * cos((2 * x + 1) * u * Math.PI / 16.0) * cos((2 * y + 1) * v * Math.PI / 16.0)) + } + } + temp[SCAN[8 * u + v]] = floor((0.25 * cu * cv * sum) + 0.5).toFloat() + } + } + return temp + } + + /** + * Generates 8 x 8 partitions of the input list. + * + * @param input List to partition. + * @param width Width of the image. + * @param height Height of the image. + * @return List of 8 x 8 partitions. + */ + private fun partition(input: List, width: Int, height: Int): ArrayList> { + val ret = ArrayList>(8 * 8) + for (i in 0 until 8 * 8) { + ret.add(LinkedList()) + } + + for ((i, t) in input.withIndex()) { + val index = (((i % width) * 8) / width) + 8 * (i * 8 / width / height) + ret[index].add(t) + } + + return ret + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/cld/CLDExtractor.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/cld/CLDExtractor.kt new file mode 100644 index 000000000..68e128570 --- /dev/null +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/cld/CLDExtractor.kt @@ -0,0 +1,44 @@ +package org.vitrivr.engine.module.features.feature.cld + +import org.vitrivr.engine.core.features.AbstractExtractor +import org.vitrivr.engine.core.features.metadata.source.file.FileSourceMetadataExtractor +import org.vitrivr.engine.core.model.content.ContentType +import org.vitrivr.engine.core.model.content.element.ImageContent +import org.vitrivr.engine.core.model.descriptor.Descriptor +import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.retrievable.Retrievable +import org.vitrivr.engine.core.operators.Operator +import org.vitrivr.engine.core.operators.ingest.Extractor +import org.vitrivr.engine.core.source.file.FileSource + +/** + * [Extractor] implementation for the [CLD] analyser. + * + * @see [CLD] + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class CLDExtractor(input: Operator, analyser: CLD, field: Schema.Field?) : AbstractExtractor(input, analyser, field) { + /** + * Internal method to check, if [Retrievable] matches this [Extractor] and should thus be processed. + * + * [FileSourceMetadataExtractor] implementation only works with [Retrievable] that contain a [FileSource]. + * + * @param retrievable The [Retrievable] to check. + * @return True on match, false otherwise, + */ + override fun matches(retrievable: Retrievable): Boolean = retrievable.content.any { it.type == ContentType.BITMAP_IMAGE } + + /** + * Internal method to perform extraction on [Retrievable]. + * + * @param retrievable The [Retrievable] to process. + * @return List of resulting [Descriptor]s. + */ + override fun extract(retrievable: Retrievable): List { + val content = retrievable.content.filterIsInstance() + return content.map { (this.analyser as CLD).analyse(it).copy(retrievableId = retrievable.id, field = this.field) } + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/dominantcolor/DominantColor.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/dominantcolor/DominantColor.kt new file mode 100644 index 000000000..f102b92ff --- /dev/null +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/dominantcolor/DominantColor.kt @@ -0,0 +1,145 @@ +package org.vitrivr.engine.module.features.feature.dominantcolor + +import org.vitrivr.engine.core.context.IndexContext +import org.vitrivr.engine.core.context.QueryContext +import org.vitrivr.engine.core.model.color.RGBColorContainer +import org.vitrivr.engine.core.model.content.element.ImageContent +import org.vitrivr.engine.core.model.descriptor.struct.LabelDescriptor +import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor +import org.vitrivr.engine.core.model.metamodel.Analyser +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.query.Query +import org.vitrivr.engine.core.model.query.bool.BooleanQuery +import org.vitrivr.engine.core.model.query.bool.SimpleBooleanQuery +import org.vitrivr.engine.core.model.retrievable.Retrievable +import org.vitrivr.engine.core.model.types.Value +import org.vitrivr.engine.core.operators.Operator +import org.vitrivr.engine.core.operators.ingest.Extractor +import org.vitrivr.engine.core.operators.retrieve.Retriever +import org.vitrivr.engine.core.util.extension.getRGBArray +import java.util.* + +class DominantColor : Analyser { + override val contentClasses = setOf(ImageContent::class) + override val descriptorClass = LabelDescriptor::class + + enum class ColorLabel { + UNDETERMINED, + BLACKWHITE, + RED, + ORANGE, + YELLOW, + GREEN, + CYAN, + BLUE, + VIOLET, + MAGENTA, + GRAY + } + + /** + * Generates a prototypical [FloatVectorDescriptor] for this [DominantColor]. + * + * @param field [Schema.Field] to create the prototype for. + * @return [FloatVectorDescriptor] + */ + override fun prototype(field: Schema.Field<*, *>) = LabelDescriptor(UUID.randomUUID(), UUID.randomUUID(), "") + + override fun newRetrieverForQuery( + field: Schema.Field, + query: Query, + context: QueryContext + ): Retriever { + + require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } + require(query is BooleanQuery) { "The query is not a BooleanQuery." } + + return DominantColorRetriever(field, query, context) + + } + + override fun newRetrieverForDescriptors( + field: Schema.Field, + descriptors: Collection, + context: QueryContext + ): Retriever { + + val labels = descriptors.mapNotNull { + try { + ColorLabel.valueOf(it.label.value.uppercase()) + } catch (e: IllegalArgumentException) { + null + } + }.toSet().map { it.name } + + val query = SimpleBooleanQuery(Value.String(labels.first()), attributeName = "label") + + return newRetrieverForQuery(field, query, context) + } + + override fun newRetrieverForContent( + field: Schema.Field, + content: Collection, + context: QueryContext + ): Retriever = newRetrieverForDescriptors(field, analyse(content), context) + + + + override fun newExtractor( + field: Schema.Field, + input: Operator, + context: IndexContext + ): Extractor = DominantColorExtractor(input, this, field) + + override fun newExtractor( + name: String, + input: Operator, + context: IndexContext + ): Extractor = DominantColorExtractor(input, this, null) //TODO name + + /** + * Performs the [DominantColor] analysis on the provided [List] of [ImageContent] elements. + * Adapted from Cineast. + * + * @param content The [List] of [ImageContent] elements. + * @return [List] of [LabelDescriptor]s. + */ + fun analyse(content: Collection): List = content.map { + val hist = IntArray(10) { 0 } + val rgb = it.content.getRGBArray() + rgb.forEach { color -> + val hsv = RGBColorContainer(color).toHSV() + + if(hsv.saturation < 0.02f){ + ++hist[0] + return@forEach + } + if (hsv.saturation < 0.2f || hsv.value < 0.3f) { + ++hist[9] + return@forEach + } else if (hsv.hue >= 0.07 && hsv.hue < 0.14) { //orange + ++hist[2] + } else if (hsv.hue >= 0.14 && hsv.hue < 0.17) { //yellow + ++hist[3] + } else if (hsv.hue >= 0.17 && hsv.hue < 0.44) { //green + ++hist[4] + } else if (hsv.hue >= 0.44 && hsv.hue < 0.56) { //cyan + ++hist[5] + } else if (hsv.hue >= 0.56 && hsv.hue < 0.73) { //blue + ++hist[6] + } else if (hsv.hue >= 0.73 && hsv.hue < 0.76) { //violet + ++hist[7] + } else if (hsv.hue >= 0.76 && hsv.hue < 0.92) { //magenta + ++hist[8] + } else { + ++hist[1] + } + + } + + val max = hist.max() + val label = if (max < rgb.size / 2) ColorLabel.UNDETERMINED else ColorLabel.entries[hist.indexOf(max)] + + return@map LabelDescriptor(UUID.randomUUID(), UUID.randomUUID(), label.name) + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/dominantcolor/DominantColorExtractor.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/dominantcolor/DominantColorExtractor.kt new file mode 100644 index 000000000..69743a151 --- /dev/null +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/dominantcolor/DominantColorExtractor.kt @@ -0,0 +1,34 @@ +package org.vitrivr.engine.module.features.feature.dominantcolor + +import org.vitrivr.engine.core.features.AbstractExtractor +import org.vitrivr.engine.core.features.metadata.source.file.FileSourceMetadataExtractor +import org.vitrivr.engine.core.model.content.ContentType +import org.vitrivr.engine.core.model.content.element.ImageContent +import org.vitrivr.engine.core.model.descriptor.struct.LabelDescriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.retrievable.Retrievable +import org.vitrivr.engine.core.operators.Operator +import org.vitrivr.engine.core.operators.ingest.Extractor +import org.vitrivr.engine.core.source.file.FileSource + +class DominantColorExtractor(input: Operator, analyser: DominantColor, field: Schema.Field?) : AbstractExtractor(input, analyser, field) { + + /** + * Internal method to check, if [Retrievable] matches this [Extractor] and should thus be processed. + * + * [FileSourceMetadataExtractor] implementation only works with [Retrievable] that contain a [FileSource]. + * + * @param retrievable The [Retrievable] to check. + * @return True on match, false otherwise, + */ + override fun matches(retrievable: Retrievable): Boolean = retrievable.content.any { it.type == ContentType.BITMAP_IMAGE } + + + + override fun extract(retrievable: Retrievable): List { + val content = retrievable.content.filterIsInstance() + return (this.analyser as DominantColor).analyse(content).map { it.copy(retrievableId = retrievable.id, field = this@DominantColorExtractor.field) } + } + + +} \ No newline at end of file diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/dominantcolor/DominantColorRetriever.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/dominantcolor/DominantColorRetriever.kt new file mode 100644 index 000000000..8dce1bba5 --- /dev/null +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/dominantcolor/DominantColorRetriever.kt @@ -0,0 +1,11 @@ +package org.vitrivr.engine.module.features.feature.dominantcolor + +import org.vitrivr.engine.core.context.QueryContext +import org.vitrivr.engine.core.features.AbstractRetriever +import org.vitrivr.engine.core.model.content.element.ImageContent +import org.vitrivr.engine.core.model.descriptor.struct.LabelDescriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.query.bool.BooleanQuery + + +class DominantColorRetriever(field: Schema.Field, query: BooleanQuery, context: QueryContext) : AbstractRetriever(field, query, context) \ No newline at end of file diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/ehd/EHD.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/ehd/EHD.kt new file mode 100644 index 000000000..533770e77 --- /dev/null +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/ehd/EHD.kt @@ -0,0 +1,196 @@ +package org.vitrivr.engine.module.features.feature.ehd + +import boofcv.io.image.ConvertBufferedImage +import boofcv.struct.image.GrayU8 +import org.vitrivr.engine.core.context.IndexContext +import org.vitrivr.engine.core.context.QueryContext +import org.vitrivr.engine.core.features.dense.DenseRetriever +import org.vitrivr.engine.core.math.correspondence.LinearCorrespondence +import org.vitrivr.engine.core.model.content.Content +import org.vitrivr.engine.core.model.content.element.ContentElement +import org.vitrivr.engine.core.model.content.element.ImageContent +import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor +import org.vitrivr.engine.core.model.metamodel.Analyser +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.query.Query +import org.vitrivr.engine.core.model.query.proximity.ProximityQuery +import org.vitrivr.engine.core.model.retrievable.Retrievable +import org.vitrivr.engine.core.model.types.Value +import org.vitrivr.engine.core.operators.Operator +import org.vitrivr.engine.core.operators.ingest.Extractor +import org.vitrivr.engine.core.operators.retrieve.Retriever +import org.vitrivr.engine.core.util.math.MathHelper +import java.util.* +import kotlin.reflect.KClass + +/** + * An MPEG 7 Edge Histogram Descriptor (EHD) [Analyser] for [ImageContent] objects. + * + * Migrated from Cineast. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class EHD : Analyser { + + companion object { + private const val VECTOR_SIZE = 80 + private val MV: FloatArray = floatArrayOf(1f, -1f, 1f, -1f) + private val MH: FloatArray = floatArrayOf(1f, 1f, -1f, -1f) + private val M45: FloatArray = floatArrayOf(MathHelper.SQRT2_f, 0f, 0f, -MathHelper.SQRT2_f) + private val M135: FloatArray = floatArrayOf(0f, MathHelper.SQRT2_f, -MathHelper.SQRT2_f, 0f) + private val MN: FloatArray = floatArrayOf(2f, -2f, -2f, 2f) + + /** + * + */ + private fun edgeType(i1: Int, i2: Int, i3: Int, i4: Int): Int { + val coeffs = floatArrayOf( + MV[0] * i1 + MV[1] * i2 + MV[2] * i3 + MV[3] * i4, + MH[0] * i1 + MH[1] * i2 + MH[2] * i3 + MH[3] * i4, + M45[0] * i1 + M45[1] * i2 + M45[2] * i3 + M45[3] * i4, + M135[0] * i1 + M135[1] * i2 + M135[2] * i3 + M135[3] * i4, + MN[0] * i1 + MN[1] * i2 + MN[2] * i3 + MN[3] * i4, + ) + + var maxid = 0 + for (i in 1..4) { + if (coeffs[maxid] < coeffs[i]) { + maxid = i + } + } + + if (coeffs[maxid] >= 14) { + return maxid + } + + return -1 + } + } + + override val contentClasses: Set>> = setOf(ImageContent::class) + override val descriptorClass: KClass = FloatVectorDescriptor::class + + /** + * Generates a prototypical [FloatVectorDescriptor] for this [EHD]. + * + * @param field [Schema.Field] to create the prototype for. + * @return [FloatVectorDescriptor] + */ + override fun prototype(field: Schema.Field<*, *>): FloatVectorDescriptor = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.FloatVector(VECTOR_SIZE)) + + /** + * Generates and returns a new [EHDExtractor] instance for this [EHD]. + * + * @param name The name of the [EHDExtractor]. + * @param input The [Operator] that acts as input to the new [Extractor]. + * @param context The [IndexContext] to use with the [Extractor]. + * + * @return A new [Extractor] instance for this [Analyser] + * @throws [UnsupportedOperationException], if this [Analyser] does not support the creation of an [Extractor] instance. + */ + override fun newExtractor(name: String, input: Operator, context: IndexContext) = EHDExtractor(input, this, null) + + /** + * Generates and returns a new [EHDExtractor] instance for this [EHD]. + * + * @param field The [Schema.Field] to create an [Extractor] for. + * @param input The [Operator] that acts as input to the new [Extractor]. + * @param context The [IndexContext] to use with the [Extractor]. + * + * @return A new [Extractor] instance for this [Analyser] + * @throws [UnsupportedOperationException], if this [Analyser] does not support the creation of an [Extractor] instance. + */ + override fun newExtractor(field: Schema.Field, input: Operator, context: IndexContext) = EHDExtractor(input, this, field) + + /** + * Generates and returns a new [DenseRetriever] instance for this [EHD]. + * + * @param field The [Schema.Field] to create an [Retriever] for. + * @param query The [Query] to use with the [Retriever]. + * @param context The [QueryContext] to use with the [Retriever]. + * + * @return A new [DenseRetriever] instance for this [EHD] + */ + override fun newRetrieverForQuery(field: Schema.Field, query: Query, context: QueryContext): DenseRetriever { + require(query is ProximityQuery<*> && query.value is Value.FloatVector) { "The query is not a ProximityQuery." } + @Suppress("UNCHECKED_CAST") + return DenseRetriever(field, query as ProximityQuery, context, LinearCorrespondence(4f)) + } + + /** + * Generates and returns a new [DenseRetriever] instance for this [EHD]. + * + * Invoking this method involves converting the provided [FloatVectorDescriptor] into a [ProximityQuery] that can be used to retrieve similar [ImageContent] elements. + * + * @param field The [Schema.Field] to create an [Retriever] for. + * @param descriptors An array of [FloatVectorDescriptor] elements to use with the [Retriever] + * @param context The [QueryContext] to use with the [Retriever] + */ + override fun newRetrieverForDescriptors(field: Schema.Field, descriptors: Collection, context: QueryContext): DenseRetriever { + /* Prepare query parameters. */ + val k = context.getProperty(field.fieldName, "limit")?.toLongOrNull() ?: 1000L + val fetchVector = context.getProperty(field.fieldName, "returnDescriptor")?.toBooleanStrictOrNull() ?: false + + /* Return retriever. */ + return this.newRetrieverForQuery(field, ProximityQuery(value = descriptors.first().vector, k = k, fetchVector = fetchVector), context) + } + + /** + * Generates and returns a new [DenseRetriever] instance for this [EHD]. + * + * Invoking this method involves converting the provided [ImageContent] and the [QueryContext] into a [FloatVectorDescriptor] + * that can be used to retrieve similar [ImageContent] elements. + * + * @param field The [Schema.Field] to create an [Retriever] for. + * @param content An array of [Content] elements to use with the [Retriever] + * @param context The [QueryContext] to use with the [Retriever] + */ + override fun newRetrieverForContent(field: Schema.Field, content: Collection, context: QueryContext) = this.newRetrieverForDescriptors(field, content.map { this.analyse(it) }, context) + + /** + * Performs the [EHD] analysis on the provided [ImageContent] and returns a [FloatVectorDescriptor] that represents the result. + * + * @param content [ImageContent] to be analysed. + * @return [FloatVectorDescriptor] result of the analysis. + */ + fun analyse(content: ImageContent): FloatVectorDescriptor { + val gray: GrayU8 = ConvertBufferedImage.convertFrom(content.content, null as GrayU8?) + val width: Int = content.content.width + val height: Int = content.content.height + val hist = FloatArray(VECTOR_SIZE) + for (x in 0..3) { + for (y in 0..3) { + val subImage: GrayU8 = gray.subimage( + width * x / 4, height * y / 4, width * (x + 1) / 4, height * (y + 1) / 4, + null + ) + var count = 0 + val tmp = IntArray(5) + var xx = 0 + while (xx < subImage.getWidth() - 1) { + var yy = 0 + while (yy < subImage.getHeight() - 1) { + count++ + val index: Int = edgeType( + subImage.unsafe_get(xx, yy), + subImage.unsafe_get(xx + 1, yy), + subImage.unsafe_get(xx, yy + 1), + subImage.unsafe_get(xx + 1, yy + 1) + ) + if (index > -1) { + tmp[index]++ + } + yy += 2 + } + xx += 2 + } + val offset = (4 * x + y) * 5 + for (i in 0..4) { + hist[offset + i] += (tmp[i].toFloat()) / count.toFloat() + } + } + } + return FloatVectorDescriptor(vector = Value.FloatVector(hist)) + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/ehd/EHDExtractor.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/ehd/EHDExtractor.kt new file mode 100644 index 000000000..3de500929 --- /dev/null +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/ehd/EHDExtractor.kt @@ -0,0 +1,44 @@ +package org.vitrivr.engine.module.features.feature.ehd + +import org.vitrivr.engine.core.features.AbstractExtractor +import org.vitrivr.engine.core.features.metadata.source.file.FileSourceMetadataExtractor +import org.vitrivr.engine.core.model.content.ContentType +import org.vitrivr.engine.core.model.content.element.ImageContent +import org.vitrivr.engine.core.model.descriptor.Descriptor +import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.retrievable.Retrievable +import org.vitrivr.engine.core.operators.Operator +import org.vitrivr.engine.core.operators.ingest.Extractor +import org.vitrivr.engine.core.source.file.FileSource + +/** + * [Extractor] implementation for the [EHD] analyser. + * + * @see [EHD] + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class EHDExtractor(input: Operator, analyser: EHD, field: Schema.Field?) : AbstractExtractor(input, analyser, field) { + /** + * Internal method to check, if [Retrievable] matches this [Extractor] and should thus be processed. + * + * [FileSourceMetadataExtractor] implementation only works with [Retrievable] that contain a [FileSource]. + * + * @param retrievable The [Retrievable] to check. + * @return True on match, false otherwise, + */ + override fun matches(retrievable: Retrievable): Boolean = retrievable.content.any { it.type == ContentType.BITMAP_IMAGE } + + /** + * Internal method to perform extraction on [Retrievable]. + * + * @param retrievable The [Retrievable] to process. + * @return List of resulting [Descriptor]s. + */ + override fun extract(retrievable: Retrievable): List { + val content = retrievable.content.filterIsInstance() + return content.map { (this.analyser as EHD).analyse(it).copy(retrievableId = retrievable.id, field = this.field) } + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/external/implementations/clip/CLIP.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/external/implementations/clip/CLIP.kt index 6ef5eab20..ab4f3dcd4 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/external/implementations/clip/CLIP.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/external/implementations/clip/CLIP.kt @@ -3,6 +3,7 @@ package org.vitrivr.engine.module.features.feature.external.implementations.clip import org.vitrivr.engine.core.context.IndexContext import org.vitrivr.engine.core.context.QueryContext import org.vitrivr.engine.core.features.dense.DenseRetriever +import org.vitrivr.engine.core.math.correspondence.BoundedCorrespondence import org.vitrivr.engine.core.model.content.element.ContentElement import org.vitrivr.engine.core.model.content.element.ImageContent import org.vitrivr.engine.core.model.content.element.TextContent @@ -106,10 +107,9 @@ class CLIP : ExternalAnalyser, FloatVectorDescriptor>() { * @throws [UnsupportedOperationException], if this [CLIP] does not support the creation of an [Retriever] instance. */ override fun newRetrieverForQuery(field: Schema.Field, FloatVectorDescriptor>, query: Query, context: QueryContext): DenseRetriever> { - require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } require(query is ProximityQuery<*> && query.value is Value.FloatVector) { "The query is not a ProximityQuery." } @Suppress("UNCHECKED_CAST") - return DenseRetriever(field, query as ProximityQuery, context) + return DenseRetriever(field, query as ProximityQuery, context, BoundedCorrespondence(0.0f, 2.0f)) } /** diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/external/implementations/dino/DINO.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/external/implementations/dino/DINO.kt index 34fe388a8..e23f1f19b 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/external/implementations/dino/DINO.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/external/implementations/dino/DINO.kt @@ -3,6 +3,7 @@ package org.vitrivr.engine.module.features.feature.external.implementations.dino import org.vitrivr.engine.core.context.IndexContext import org.vitrivr.engine.core.context.QueryContext import org.vitrivr.engine.core.features.dense.DenseRetriever +import org.vitrivr.engine.core.math.correspondence.BoundedCorrespondence import org.vitrivr.engine.core.model.content.Content import org.vitrivr.engine.core.model.content.element.ContentElement import org.vitrivr.engine.core.model.content.element.ImageContent @@ -93,10 +94,9 @@ class DINO : ExternalAnalyser() { * @throws [UnsupportedOperationException], if this [Analyser] does not support the creation of an [Retriever] instance. */ override fun newRetrieverForQuery(field: Schema.Field, query: Query, context: QueryContext): DenseRetriever { - require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } require(query is ProximityQuery<*> && query.value is Value.FloatVector) { "The query is not a ProximityQuery." } @Suppress("UNCHECKED_CAST") - return DenseRetriever(field, query as ProximityQuery, context) + return DenseRetriever(field, query as ProximityQuery, context, BoundedCorrespondence(0.0f, 2.0f)) } /** diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogram.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogram.kt new file mode 100644 index 000000000..27c8ba374 --- /dev/null +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogram.kt @@ -0,0 +1,153 @@ +package org.vitrivr.engine.module.features.feature.huehistogram + +import org.vitrivr.engine.core.context.IndexContext +import org.vitrivr.engine.core.context.QueryContext +import org.vitrivr.engine.core.features.dense.DenseRetriever +import org.vitrivr.engine.core.math.correspondence.LinearCorrespondence +import org.vitrivr.engine.core.model.color.HSVColorContainer +import org.vitrivr.engine.core.model.color.RGBColorContainer +import org.vitrivr.engine.core.model.content.Content +import org.vitrivr.engine.core.model.content.element.ContentElement +import org.vitrivr.engine.core.model.content.element.ImageContent +import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor +import org.vitrivr.engine.core.model.metamodel.Analyser +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.query.Query +import org.vitrivr.engine.core.model.query.proximity.ProximityQuery +import org.vitrivr.engine.core.model.retrievable.Retrievable +import org.vitrivr.engine.core.model.types.Value +import org.vitrivr.engine.core.operators.Operator +import org.vitrivr.engine.core.operators.ingest.Extractor +import org.vitrivr.engine.core.operators.retrieve.Retriever +import org.vitrivr.engine.core.util.extension.getRGBArray +import java.util.* +import kotlin.reflect.KClass + +/** + * A hue histogram [Analyser] for [ImageContent] objects. + * + * Migrated from Cineast. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class HueHistogram : Analyser { + + companion object { + private const val VECTOR_SIZE = 16 + } + + override val contentClasses: Set>> = setOf(ImageContent::class) + override val descriptorClass: KClass = FloatVectorDescriptor::class + + /** + * Generates a prototypical [FloatVectorDescriptor] for this [HueHistogram]. + * + * @param field [Schema.Field] to create the prototype for. + * @return [FloatVectorDescriptor] + */ + override fun prototype(field: Schema.Field<*, *>): FloatVectorDescriptor = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.FloatVector(VECTOR_SIZE)) + + /** + * Generates and returns a new [HueHistogramExtractor] instance for this [HueHistogram]. + * + * @param name The name of the [HueHistogramExtractor]. + * @param input The [Operator] that acts as input to the new [Extractor]. + * @param context The [IndexContext] to use with the [Extractor]. + * + * @return A new [Extractor] instance for this [Analyser] + * @throws [UnsupportedOperationException], if this [Analyser] does not support the creation of an [Extractor] instance. + */ + override fun newExtractor(name: String, input: Operator, context: IndexContext) = HueHistogramExtractor(input, this, null) + + /** + * Generates and returns a new [HueHistogramExtractor] instance for this [HueHistogram]. + * + * @param field The [Schema.Field] to create an [Extractor] for. + * @param input The [Operator] that acts as input to the new [Extractor]. + * @param context The [IndexContext] to use with the [Extractor]. + * + * @return A new [Extractor] instance for this [Analyser] + * @throws [UnsupportedOperationException], if this [Analyser] does not support the creation of an [Extractor] instance. + */ + override fun newExtractor(field: Schema.Field, input: Operator, context: IndexContext) = HueHistogramExtractor(input, this, field) + + /** + * Generates and returns a new [DenseRetriever] instance for this [HueHistogram]. + * + * @param field The [Schema.Field] to create an [Retriever] for. + * @param query The [Query] to use with the [Retriever]. + * @param context The [QueryContext] to use with the [Retriever]. + * + * @return A new [DenseRetriever] instance for this [HueHistogram] + */ + override fun newRetrieverForQuery(field: Schema.Field, query: Query, context: QueryContext): DenseRetriever { + require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } + require(query is ProximityQuery<*> && query.value is Value.FloatVector) { "The query is not a ProximityQuery." } + @Suppress("UNCHECKED_CAST") + return DenseRetriever(field, query as ProximityQuery, context, LinearCorrespondence(16f)) + } + + /** + * Generates and returns a new [DenseRetriever] instance for this [HueHistogram]. + * + * Invoking this method involves converting the provided [FloatVectorDescriptor] into a [ProximityQuery] that can be used to retrieve similar [ImageContent] elements. + * + * @param field The [Schema.Field] to create an [Retriever] for. + * @param descriptors An array of [FloatVectorDescriptor] elements to use with the [Retriever] + * @param context The [QueryContext] to use with the [Retriever] + */ + override fun newRetrieverForDescriptors(field: Schema.Field, descriptors: Collection, context: QueryContext): DenseRetriever { + require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } + + /* Prepare query parameters. */ + val k = context.getProperty(field.fieldName, "limit")?.toLongOrNull() ?: 1000L + val fetchVector = context.getProperty(field.fieldName, "returnDescriptor")?.toBooleanStrictOrNull() ?: false + + /* Return retriever. */ + return this.newRetrieverForQuery(field, ProximityQuery(value = descriptors.first().vector, k = k, fetchVector = fetchVector), context) + } + + /** + * Generates and returns a new [DenseRetriever] instance for this [HueHistogram]. + * + * Invoking this method involves converting the provided [ImageContent] and the [QueryContext] into a [FloatVectorDescriptor] + * that can be used to retrieve similar [ImageContent] elements. + * + * @param field The [Schema.Field] to create an [Retriever] for. + * @param content An array of [Content] elements to use with the [Retriever] + * @param context The [QueryContext] to use with the [Retriever] + */ + override fun newRetrieverForContent(field: Schema.Field, content: Collection, context: QueryContext) = this.newRetrieverForDescriptors(field, content.map { this.analyse(it) }, context) + + /** + * Performs the [HueHistogram] analysis on the provided [ImageContent] and returns a [FloatVectorDescriptor] that represents the result. + * + * @param content [ImageContent] to be analysed. + * @return [FloatVectorDescriptor] result of the analysis. + */ + fun analyse(content: ImageContent): FloatVectorDescriptor { + /* Generate histogram. */ + val hist = FloatArray(VECTOR_SIZE) + val colors = content.content.getRGBArray() + for (color in colors) { + val container: HSVColorContainer = RGBColorContainer(color).toHSV() + if (container.saturation > 0.2f && container.value > 0.3f) { + val h: Float = container.hue * hist.size + val idx = h.toInt() + hist[idx] += h - idx + hist[(idx + 1) % hist.size] += idx + 1 - h + } + } + + /* Normalize hist. */ + val sum = hist.sum() + if (sum > 1f) { + for (i in hist.indices) { + hist[i] /= sum + } + } + + return FloatVectorDescriptor(vector = Value.FloatVector(hist)) + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogramExtractor.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogramExtractor.kt new file mode 100644 index 000000000..f16974cb5 --- /dev/null +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogramExtractor.kt @@ -0,0 +1,45 @@ +package org.vitrivr.engine.module.features.feature.huehistogram + +import org.vitrivr.engine.core.features.AbstractExtractor +import org.vitrivr.engine.core.features.metadata.source.file.FileSourceMetadataExtractor +import org.vitrivr.engine.core.model.content.ContentType +import org.vitrivr.engine.core.model.content.element.ImageContent +import org.vitrivr.engine.core.model.descriptor.Descriptor +import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.retrievable.Retrievable +import org.vitrivr.engine.core.operators.Operator +import org.vitrivr.engine.core.operators.ingest.Extractor +import org.vitrivr.engine.core.source.file.FileSource +import org.vitrivr.engine.module.features.feature.ehd.EHD + +/** + * An [Extractor] implementation for the [HueHistogram] analyser. + * + * @see [EHD] + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class HueHistogramExtractor(input: Operator, analyser: HueHistogram, field: Schema.Field?) : AbstractExtractor(input, analyser, field) { + /** + * Internal method to check, if [Retrievable] matches this [Extractor] and should thus be processed. + * + * [FileSourceMetadataExtractor] implementation only works with [Retrievable] that contain a [FileSource]. + * + * @param retrievable The [Retrievable] to check. + * @return True on match, false otherwise, + */ + override fun matches(retrievable: Retrievable): Boolean = retrievable.content.any { it.type == ContentType.BITMAP_IMAGE } + + /** + * Internal method to perform extraction on [Retrievable]. + * + * @param retrievable The [Retrievable] to process. + * @return List of resulting [Descriptor]s. + */ + override fun extract(retrievable: Retrievable): List { + val content = retrievable.content.filterIsInstance() + return content.map { (this.analyser as HueHistogram).analyse(it).copy(retrievableId = retrievable.id, field = this.field) } + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/mediancolor/MedianColor.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/mediancolor/MedianColor.kt new file mode 100644 index 000000000..b95507a00 --- /dev/null +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/mediancolor/MedianColor.kt @@ -0,0 +1,144 @@ +package org.vitrivr.engine.module.features.feature.mediancolor + +import org.vitrivr.engine.core.context.IndexContext +import org.vitrivr.engine.core.context.QueryContext +import org.vitrivr.engine.core.features.dense.DenseRetriever +import org.vitrivr.engine.core.math.correspondence.LinearCorrespondence +import org.vitrivr.engine.core.model.color.RGBColorContainer +import org.vitrivr.engine.core.model.content.Content +import org.vitrivr.engine.core.model.content.element.ImageContent +import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor +import org.vitrivr.engine.core.model.metamodel.Analyser +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.query.Query +import org.vitrivr.engine.core.model.query.proximity.ProximityQuery +import org.vitrivr.engine.core.model.retrievable.Retrievable +import org.vitrivr.engine.core.model.types.Value +import org.vitrivr.engine.core.operators.Operator +import org.vitrivr.engine.core.operators.ingest.Extractor +import org.vitrivr.engine.core.operators.retrieve.Retriever +import org.vitrivr.engine.core.util.extension.getRGBArray +import org.vitrivr.engine.core.util.math.StatisticsHelper.medianFromHistogram +import java.util.* + +/** + * Implementation of the [MedianColor] [Analyser], which derives the median color from an [ImageContent] as [FloatVectorDescriptor]. + * + * This [Analyser] has little practical relevance these days but acts as a simple example for how to create a custom [Analyser] that uses vectors. + * Furthermore, it can be used to implement test cases. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class MedianColor : Analyser { + + companion object { + private const val VECTOR_SIZE = 3 + } + + override val contentClasses = setOf(ImageContent::class) + override val descriptorClass = FloatVectorDescriptor::class + + /** + * Generates a prototypical [FloatVectorDescriptor] for this [MedianColor]. + * + * @param field [Schema.Field] to create the prototype for. + * @return [FloatVectorDescriptor] + */ + override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.FloatVector(VECTOR_SIZE)) + + /** + * Generates and returns a new [MedianColorExtractor] instance for this [MedianColor]. + * + * @param field The [Schema.Field] to create an [Extractor] for. + * @param input The [Operator] that acts as input to the new [Extractor]. + * @param context The [IndexContext] to use with the [Extractor]. + * + * @return A new [Extractor] instance for this [Analyser] + * @throws [UnsupportedOperationException], if this [Analyser] does not support the creation of an [Extractor] instance. + */ + override fun newExtractor(field: Schema.Field, input: Operator, context: IndexContext) = MedianColorExtractor(input, this, field) + + /** + * Generates and returns a new [MedianColorExtractor] instance for this [MedianColor]. + * + * @param name The name of the [MedianColorExtractor]. + * @param input The [Operator] that acts as input to the new [Extractor]. + * @param context The [IndexContext] to use with the [Extractor]. + * + * @return A new [Extractor] instance for this [Analyser] + * @throws [UnsupportedOperationException], if this [Analyser] does not support the creation of an [Extractor] instance. + */ + override fun newExtractor(name: String, input: Operator, context: IndexContext): Extractor = MedianColorExtractor(input, this, null) + + /** + * Generates and returns a new [DenseRetriever] instance for this [MedianColor]. + * + * @param field The [Schema.Field] to create an [Retriever] for. + * @param query The [Query] to use with the [Retriever]. + * @param context The [QueryContext] to use with the [Retriever]. + * + * @return A new [Retriever] instance for this [Analyser] + */ + override fun newRetrieverForQuery(field: Schema.Field, query: Query, context: QueryContext): DenseRetriever { + require(query is ProximityQuery<*> && query.value is Value.FloatVector) { "The query is not a ProximityQuery." } + @Suppress("UNCHECKED_CAST") + return DenseRetriever(field, query as ProximityQuery, context, LinearCorrespondence(3f)) + } + + /** + * Generates and returns a new [DenseRetriever] instance for this [MedianColor]. + * + * Invoking this method involves converting the provided [FloatVectorDescriptor] into a [ProximityQuery] that can be used to retrieve similar [ImageContent] elements. + * + * @param field The [Schema.Field] to create an [Retriever] for. + * @param descriptors An array of [FloatVectorDescriptor] elements to use with the [Retriever] + * @param context The [QueryContext] to use with the [Retriever] + */ + override fun newRetrieverForDescriptors(field: Schema.Field, descriptors: Collection, context: QueryContext): DenseRetriever { + /* Prepare query parameters. */ + val k = context.getProperty(field.fieldName, "limit")?.toLongOrNull() ?: 1000L + val fetchVector = context.getProperty(field.fieldName, "returnDescriptor")?.toBooleanStrictOrNull() ?: false + + /* Return retriever. */ + return this.newRetrieverForQuery(field, ProximityQuery(value = descriptors.first().vector, k = k, fetchVector = fetchVector), context) + } + + /** + * Generates and returns a new [DenseRetriever] instance for this [MedianColor]. + * + * Invoking this method involves converting the provided [ImageContent] and the [QueryContext] into a [FloatVectorDescriptor] + * that can be used to retrieve similar [ImageContent] elements. + * + * @param field The [Schema.Field] to create an [Retriever] for. + * @param content An array of [Content] elements to use with the [Retriever] + * @param context The [QueryContext] to use with the [Retriever] + */ + override fun newRetrieverForContent(field: Schema.Field, content: Collection, context: QueryContext): DenseRetriever = + this.newRetrieverForDescriptors(field, content.map { this.analyse(it) }, context) + + /** + * Performs the [MedianColor] analysis on the provided [List] of [ImageContent] elements. + * + * @param content The [List] of [ImageContent] elements. + * @return [FloatVectorDescriptor]s. + */ + fun analyse(content: ImageContent): FloatVectorDescriptor { + val r = IntArray(256) + val g = IntArray(256) + val b = IntArray(256) + + /* Extract colors from content and generate histogram. */ + val colors: IntArray = content.content.getRGBArray() + for (color in colors) { + val rgb = RGBColorContainer(color) + r[rgb.redAsInt()]++ + g[rgb.greenAsInt()]++ + b[rgb.blueAsInt()]++ + } + + /* Generate vector from per-color histograms. */ + val lab = RGBColorContainer(medianFromHistogram(r), medianFromHistogram(g), medianFromHistogram(b)).toLab() + return FloatVectorDescriptor(vector = lab.toVector()) + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/mediancolor/MedianColorExtractor.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/mediancolor/MedianColorExtractor.kt new file mode 100644 index 000000000..1773db968 --- /dev/null +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/mediancolor/MedianColorExtractor.kt @@ -0,0 +1,45 @@ +package org.vitrivr.engine.module.features.feature.mediancolor + +import org.vitrivr.engine.core.features.AbstractExtractor +import org.vitrivr.engine.core.features.metadata.source.file.FileSourceMetadataExtractor +import org.vitrivr.engine.core.model.content.ContentType +import org.vitrivr.engine.core.model.content.element.ImageContent +import org.vitrivr.engine.core.model.descriptor.Descriptor +import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.retrievable.Retrievable +import org.vitrivr.engine.core.operators.Operator +import org.vitrivr.engine.core.operators.ingest.Extractor +import org.vitrivr.engine.core.source.file.FileSource + + +/** + * [Extractor] implementation for the [MedianColor] analyser. + * + * @see [MedianColor] + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class MedianColorExtractor(input: Operator, analyser: MedianColor, field: Schema.Field?) : AbstractExtractor(input, analyser, field) { + /** + * Internal method to check, if [Retrievable] matches this [Extractor] and should thus be processed. + * + * [FileSourceMetadataExtractor] implementation only works with [Retrievable] that contain a [FileSource]. + * + * @param retrievable The [Retrievable] to check. + * @return True on match, false otherwise, + */ + override fun matches(retrievable: Retrievable): Boolean = retrievable.content.any { it.type == ContentType.BITMAP_IMAGE } + + /** + * Internal method to perform extraction on [Retrievable]. + ** + * @param retrievable The [Retrievable] to process. + * @return List of resulting [Descriptor]s. + */ + override fun extract(retrievable: Retrievable): List { + val content = retrievable.content.filterIsInstance() + return content.map { (this.analyser as MedianColor).analyse(it).copy(retrievableId = retrievable.id, field = this.field) } + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser b/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser index d4ef3f422..2aea6b065 100644 --- a/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser +++ b/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser @@ -1,2 +1,7 @@ +org.vitrivr.engine.module.features.feature.cld.CLD +org.vitrivr.engine.module.features.feature.dominantcolor.DominantColor +org.vitrivr.engine.module.features.feature.ehd.EHD +org.vitrivr.engine.module.features.feature.huehistogram.HueHistogram +org.vitrivr.engine.module.features.feature.mediancolor.MedianColor org.vitrivr.engine.module.features.feature.external.implementations.dino.DINO -org.vitrivr.engine.module.features.feature.external.implementations.clip.CLIP +org.vitrivr.engine.module.features.feature.external.implementations.clip.CLIP \ No newline at end of file diff --git a/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/api/ImageCaptioningApi.kt b/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/api/ImageCaptioningApi.kt index a2602c5e6..a3a073e73 100644 --- a/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/api/ImageCaptioningApi.kt +++ b/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/api/ImageCaptioningApi.kt @@ -82,7 +82,7 @@ class ImageCaptioningApi(host: String, model: String, timeoutMs: Long, pollingIn */ override suspend fun pollBatchedJob(jobId: String): JobResult> = try { this.imageCaptioningApi.getBatchedJobResultsApiTasksImageCaptioningBatchedJobsJobGet(jobId).body().let { result -> - val values = result.result?.map { Value.Text(it.caption.trim() ?: "") } + val values = result.result?.map { Value.Text(it.caption.trim()) } JobResult(result.status, values) } } catch (e: Throwable) { diff --git a/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/dense/DenseEmbedding.kt b/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/dense/DenseEmbedding.kt index 2be8f709b..ee420a0eb 100644 --- a/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/dense/DenseEmbedding.kt +++ b/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/dense/DenseEmbedding.kt @@ -10,6 +10,7 @@ import org.vitrivr.engine.base.features.external.implementations.asr.ASRExtracto import org.vitrivr.engine.core.context.IndexContext import org.vitrivr.engine.core.context.QueryContext import org.vitrivr.engine.core.features.dense.DenseRetriever +import org.vitrivr.engine.core.math.correspondence.BoundedCorrespondence import org.vitrivr.engine.core.model.content.element.ContentElement import org.vitrivr.engine.core.model.content.element.ImageContent import org.vitrivr.engine.core.model.content.element.TextContent @@ -83,7 +84,7 @@ class DenseEmbedding : ExternalFesAnalyser, FloatVectorDescrip require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } require(query is ProximityQuery<*> && query.value is Value.FloatVector) { "The query is not a ProximityQuery." } @Suppress("UNCHECKED_CAST") - return DenseRetriever(field, query as ProximityQuery, context) + return DenseRetriever(field, query as ProximityQuery, context, BoundedCorrespondence(0.0f, 2.0f)) } /**