-
Notifications
You must be signed in to change notification settings - Fork 3
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Adds HueHistogram feature adapted from Cineast.
Signed-off-by: Ralph Gasser <[email protected]>
- Loading branch information
1 parent
d1c29c3
commit 2edf650
Showing
13 changed files
with
322 additions
and
11 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
44 changes: 44 additions & 0 deletions
44
vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/ColorConverter.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,44 @@ | ||
| package org.vitrivr.engine.core.model.color | ||
|
|
||
| import org.vitrivr.engine.core.model.color.hsv.HSVColorContainer | ||
| import org.vitrivr.engine.core.model.color.rgb.RGBByteColorContainer | ||
| import org.vitrivr.engine.core.model.color.rgb.RGBFloatColorContainer | ||
|
|
||
| /** | ||
| * A color converter that provides methods to convert between different color spaces. | ||
| * | ||
| * Adapted from Cineast. | ||
| * | ||
| * @author Ralph Gasser | ||
| * @version 1.0.0 | ||
| */ | ||
| object ColorConverter { | ||
|
|
||
| /** | ||
| * Converts a [RGBByteColorContainer] to a [HSVColorContainer]. | ||
| * | ||
| * @param rgb The [RGBByteColorContainer] to convert. | ||
| * @return [HSVColorContainer] | ||
| */ | ||
| fun rgbToHsv(rgb: RGBByteColorContainer): HSVColorContainer = rgbToHsv(rgb.toFloatContainer()) | ||
|
|
||
| /** | ||
| * Converts a [RGBFloatColorContainer] to a [HSVColorContainer]. | ||
| * | ||
| * @param rgb The [RGBFloatColorContainer] to convert. | ||
| * @return [HSVColorContainer] | ||
| */ | ||
| fun rgbToHsv(rgb: RGBFloatColorContainer): HSVColorContainer { | ||
| val max = maxOf(rgb.red, rgb.green, rgb.blue) | ||
| val min = minOf(rgb.red, rgb.green, rgb.blue) | ||
| val d = max - min | ||
| val s = if (max == 0f) 0f else d / max | ||
| val h = when { | ||
| d == 0f -> 0f | ||
| max == rgb.red -> (rgb.green - rgb.blue) / d + (if (rgb.green < rgb.blue) 6 else 0) | ||
| max == rgb.green -> (rgb.blue - rgb.red) / d + 2 | ||
| else -> (rgb.red - rgb.green) / d + 4 | ||
| } | ||
| return HSVColorContainer(h, s, max) | ||
| } | ||
| } |
26 changes: 26 additions & 0 deletions
26
...-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/hsv/HSVColorContainer.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| package org.vitrivr.engine.core.model.color.hsv | ||
|
|
||
| /** | ||
| * A container for HSV colors. | ||
| * | ||
| * @author Ralph Gasser | ||
| * @version 1.0.0 | ||
| */ | ||
| @JvmInline | ||
| value class HSVColorContainer private constructor(private val colors: FloatArray) { | ||
|
|
||
| constructor(hue: Float, saturation: Float, value: Float) : this(floatArrayOf(hue, saturation, value)) | ||
| constructor(hue: Double, saturation: Double, value: Double) : this(floatArrayOf(hue.toFloat(), saturation.toFloat(), value.toFloat())) | ||
|
|
||
| /** Accessor for the hue component of the [HSVColorContainer]. */ | ||
| val hue: Float | ||
| get() = this.colors[0] | ||
|
|
||
| /** Accessor for the saturation component of the [HSVColorContainer]. */ | ||
| val saturation: Float | ||
| get() = this.colors[1] | ||
|
|
||
| /** Accessor for the value component of the [HSVColorContainer]. */ | ||
| val value: Float | ||
| get() = this.colors[2] | ||
| } |
2 changes: 1 addition & 1 deletion
2
...el/color/MutableRGBFloatColorContainer.kt → ...olor/rgb/MutableRGBFloatColorContainer.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
2 changes: 1 addition & 1 deletion
2
...ore/model/color/RGBFloatColorContainer.kt → ...model/color/rgb/RGBFloatColorContainer.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
151 changes: 151 additions & 0 deletions
151
...s/src/main/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogram.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,151 @@ | ||
| package org.vitrivr.engine.module.features.feature.huehistogram | ||
|
|
||
| import org.vitrivr.engine.core.context.IndexContext | ||
| import org.vitrivr.engine.core.context.QueryContext | ||
| import org.vitrivr.engine.core.model.color.ColorConverter | ||
| import org.vitrivr.engine.core.model.color.hsv.HSVColorContainer | ||
| import org.vitrivr.engine.core.model.color.rgb.RGBByteColorContainer | ||
| import org.vitrivr.engine.core.model.content.Content | ||
| import org.vitrivr.engine.core.model.content.element.ContentElement | ||
| import org.vitrivr.engine.core.model.content.element.ImageContent | ||
| import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor | ||
| import org.vitrivr.engine.core.model.metamodel.Analyser | ||
| import org.vitrivr.engine.core.model.metamodel.Schema | ||
| import org.vitrivr.engine.core.model.query.Query | ||
| import org.vitrivr.engine.core.model.query.proximity.ProximityQuery | ||
| import org.vitrivr.engine.core.model.retrievable.Retrievable | ||
| import org.vitrivr.engine.core.model.types.Value | ||
| import org.vitrivr.engine.core.operators.Operator | ||
| import org.vitrivr.engine.core.operators.ingest.Extractor | ||
| import org.vitrivr.engine.core.operators.retrieve.Retriever | ||
| import java.util.* | ||
| import kotlin.reflect.KClass | ||
|
|
||
| /** | ||
| * A hue histogram [Analyser] for [ImageContent] objects. | ||
| * | ||
| * Migrated from Cineast. | ||
| * | ||
| * @author Ralph Gasser | ||
| * @version 1.0.0 | ||
| */ | ||
| class HueHistogram : Analyser<ImageContent, FloatVectorDescriptor> { | ||
|
|
||
| companion object { | ||
| private const val VECTOR_SIZE = 16 | ||
| } | ||
|
|
||
| override val contentClasses: Set<KClass<out ContentElement<*>>> = setOf(ImageContent::class) | ||
| override val descriptorClass: KClass<FloatVectorDescriptor> = FloatVectorDescriptor::class | ||
|
|
||
| /** | ||
| * Generates a prototypical [FloatVectorDescriptor] for this [HueHistogram]. | ||
| * | ||
| * @param field [Schema.Field] to create the prototype for. | ||
| * @return [FloatVectorDescriptor] | ||
| */ | ||
| override fun prototype(field: Schema.Field<*, *>): FloatVectorDescriptor = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.FloatVector(VECTOR_SIZE)) | ||
|
|
||
| /** | ||
| * Generates and returns a new [HueHistogramExtractor] instance for this [HueHistogram]. | ||
| * | ||
| * @param name The name of the [HueHistogramExtractor]. | ||
| * @param input The [Operator] that acts as input to the new [Extractor]. | ||
| * @param context The [IndexContext] to use with the [Extractor]. | ||
| * | ||
| * @return A new [Extractor] instance for this [Analyser] | ||
| * @throws [UnsupportedOperationException], if this [Analyser] does not support the creation of an [Extractor] instance. | ||
| */ | ||
| override fun newExtractor(name: String, input: Operator<Retrievable>, context: IndexContext) = HueHistogramExtractor(input, this, null) | ||
|
|
||
| /** | ||
| * Generates and returns a new [HueHistogramExtractor] instance for this [HueHistogram]. | ||
| * | ||
| * @param field The [Schema.Field] to create an [Extractor] for. | ||
| * @param input The [Operator] that acts as input to the new [Extractor]. | ||
| * @param context The [IndexContext] to use with the [Extractor]. | ||
| * | ||
| * @return A new [Extractor] instance for this [Analyser] | ||
| * @throws [UnsupportedOperationException], if this [Analyser] does not support the creation of an [Extractor] instance. | ||
| */ | ||
| override fun newExtractor(field: Schema.Field<ImageContent, FloatVectorDescriptor>, input: Operator<Retrievable>, context: IndexContext) = HueHistogramExtractor(input, this, field) | ||
|
|
||
| /** | ||
| * Generates and returns a new [HueHistogramRetriever] instance for this [HueHistogram]. | ||
| * | ||
| * @param field The [Schema.Field] to create an [Retriever] for. | ||
| * @param query The [Query] to use with the [Retriever]. | ||
| * @param context The [QueryContext] to use with the [Retriever]. | ||
| * | ||
| * @return A new [HueHistogramRetriever] instance for this [HueHistogram] | ||
| */ | ||
| override fun newRetrieverForQuery(field: Schema.Field<ImageContent, FloatVectorDescriptor>, query: Query, context: QueryContext): HueHistogramRetriever { | ||
| require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } | ||
| require(query is ProximityQuery<*> && query.value is Value.FloatVector) { "The query is not a ProximityQuery<Value.FloatVector>." } | ||
| @Suppress("UNCHECKED_CAST") | ||
| return HueHistogramRetriever(field, query as ProximityQuery<Value.FloatVector>, context) | ||
| } | ||
|
|
||
| /** | ||
| * Generates and returns a new [HueHistogramRetriever] instance for this [HueHistogram]. | ||
| * | ||
| * Invoking this method involves converting the provided [FloatVectorDescriptor] into a [ProximityQuery] that can be used to retrieve similar [ImageContent] elements. | ||
| * | ||
| * @param field The [Schema.Field] to create an [Retriever] for. | ||
| * @param descriptors An array of [FloatVectorDescriptor] elements to use with the [Retriever] | ||
| * @param context The [QueryContext] to use with the [Retriever] | ||
| */ | ||
| override fun newRetrieverForDescriptors(field: Schema.Field<ImageContent, FloatVectorDescriptor>, descriptors: Collection<FloatVectorDescriptor>, context: QueryContext): HueHistogramRetriever { | ||
| require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } | ||
|
|
||
| /* Prepare query parameters. */ | ||
| val k = context.getProperty(field.fieldName, "limit")?.toLongOrNull() ?: 1000L | ||
| val fetchVector = context.getProperty(field.fieldName, "returnDescriptor")?.toBooleanStrictOrNull() ?: false | ||
|
|
||
| /* Return retriever. */ | ||
| return this.newRetrieverForQuery(field, ProximityQuery(value = descriptors.first().vector, k = k, fetchVector = fetchVector), context) | ||
| } | ||
|
|
||
| /** | ||
| * Generates and returns a new [HueHistogramRetriever] instance for this [HueHistogram]. | ||
| * | ||
| * Invoking this method involves converting the provided [ImageContent] and the [QueryContext] into a [FloatVectorDescriptor] | ||
| * that can be used to retrieve similar [ImageContent] elements. | ||
| * | ||
| * @param field The [Schema.Field] to create an [Retriever] for. | ||
| * @param content An array of [Content] elements to use with the [Retriever] | ||
| * @param context The [QueryContext] to use with the [Retriever] | ||
| */ | ||
| override fun newRetrieverForContent(field: Schema.Field<ImageContent, FloatVectorDescriptor>, content: Collection<ImageContent>, context: QueryContext) = this.newRetrieverForDescriptors(field, content.map { this.analyse(it) }, context) | ||
|
|
||
| /** | ||
| * Performs the [HueHistogram] analysis on the provided [ImageContent] and returns a [FloatVectorDescriptor] that represents the result. | ||
| * | ||
| * @param content [ImageContent] to be analysed. | ||
| * @return [FloatVectorDescriptor] result of the analysis. | ||
| */ | ||
| fun analyse(content: ImageContent): FloatVectorDescriptor { | ||
| /* Generate histogram. */ | ||
| val hist = FloatArray(VECTOR_SIZE) | ||
| val colors = content.content.getRGB(0, 0, content.content.width, content.content.height, null, 0, content.content.width) | ||
| for (color in colors) { | ||
| val container: HSVColorContainer = ColorConverter.rgbToHsv(RGBByteColorContainer(color)) | ||
| if (container.saturation > 0.2f && container.value > 0.3f) { | ||
| val h: Float = container.hue * hist.size | ||
| val idx = h.toInt() | ||
| hist[idx] += h - idx | ||
| hist[(idx + 1) % hist.size] += idx + 1 - h | ||
| } | ||
| } | ||
|
|
||
| /* Normalize hist. */ | ||
| val sum = hist.sum() | ||
| if (sum > 1f) { | ||
| for (i in hist.indices) { | ||
| hist[i] /= sum | ||
| } | ||
| } | ||
|
|
||
| return FloatVectorDescriptor(vector = Value.FloatVector(hist)) | ||
| } | ||
| } |
45 changes: 45 additions & 0 deletions
45
...n/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogramExtractor.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,45 @@ | ||
| package org.vitrivr.engine.module.features.feature.huehistogram | ||
|
|
||
| import org.vitrivr.engine.core.features.AbstractExtractor | ||
| import org.vitrivr.engine.core.features.metadata.source.file.FileSourceMetadataExtractor | ||
| import org.vitrivr.engine.core.model.content.ContentType | ||
| import org.vitrivr.engine.core.model.content.element.ImageContent | ||
| import org.vitrivr.engine.core.model.descriptor.Descriptor | ||
| import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor | ||
| import org.vitrivr.engine.core.model.metamodel.Schema | ||
| import org.vitrivr.engine.core.model.retrievable.Retrievable | ||
| import org.vitrivr.engine.core.operators.Operator | ||
| import org.vitrivr.engine.core.operators.ingest.Extractor | ||
| import org.vitrivr.engine.core.source.file.FileSource | ||
| import org.vitrivr.engine.module.features.feature.ehd.EHD | ||
|
|
||
| /** | ||
| * An [Extractor] implementation for the [HueHistogram] analyser. | ||
| * | ||
| * @see [EHD] | ||
| * | ||
| * @author Ralph Gasser | ||
| * @version 1.0.0 | ||
| */ | ||
| class HueHistogramExtractor(input: Operator<Retrievable>, analyser: HueHistogram, field: Schema.Field<ImageContent, FloatVectorDescriptor>?) : AbstractExtractor<ImageContent, FloatVectorDescriptor>(input, analyser, field) { | ||
| /** | ||
| * Internal method to check, if [Retrievable] matches this [Extractor] and should thus be processed. | ||
| * | ||
| * [FileSourceMetadataExtractor] implementation only works with [Retrievable] that contain a [FileSource]. | ||
| * | ||
| * @param retrievable The [Retrievable] to check. | ||
| * @return True on match, false otherwise, | ||
| */ | ||
| override fun matches(retrievable: Retrievable): Boolean = retrievable.content.any { it.type == ContentType.BITMAP_IMAGE } | ||
|
|
||
| /** | ||
| * Internal method to perform extraction on [Retrievable]. | ||
| * | ||
| * @param retrievable The [Retrievable] to process. | ||
| * @return List of resulting [Descriptor]s. | ||
| */ | ||
| override fun extract(retrievable: Retrievable): List<FloatVectorDescriptor> { | ||
| val content = retrievable.content.filterIsInstance<ImageContent>() | ||
| return content.map { (this.analyser as HueHistogram).analyse(it).copy(retrievableId = retrievable.id, field = this.field) } | ||
| } | ||
| } |
42 changes: 42 additions & 0 deletions
42
...n/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogramRetriever.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| package org.vitrivr.engine.module.features.feature.huehistogram | ||
|
|
||
| import kotlinx.coroutines.CoroutineScope | ||
| import kotlinx.coroutines.flow.flow | ||
| import org.vitrivr.engine.core.context.QueryContext | ||
| import org.vitrivr.engine.core.features.AbstractRetriever | ||
| import org.vitrivr.engine.core.math.correspondence.LinearCorrespondence | ||
| import org.vitrivr.engine.core.model.content.element.ImageContent | ||
| import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor | ||
| import org.vitrivr.engine.core.model.metamodel.Schema | ||
| import org.vitrivr.engine.core.model.query.proximity.ProximityQuery | ||
| import org.vitrivr.engine.core.model.retrievable.attributes.DistanceAttribute | ||
| import org.vitrivr.engine.core.model.types.Value | ||
| import org.vitrivr.engine.core.operators.retrieve.Retriever | ||
|
|
||
| /** | ||
| * [Retriever] implementation for the [HueHistogram] analyser. | ||
| * | ||
| * @see [HueHistogram] | ||
| * | ||
| * @author Ralph Gasser | ||
| * @version 1.0.0 | ||
| */ | ||
| class HueHistogramRetriever(field: Schema.Field<ImageContent, FloatVectorDescriptor>, query: ProximityQuery<Value.FloatVector>, context: QueryContext) : AbstractRetriever<ImageContent, FloatVectorDescriptor>(field, query, context) { | ||
|
|
||
| companion object { | ||
| /** [LinearCorrespondence] for [HueHistogramRetriever]. Maximum distance is taken from Cineast implementation. */ | ||
| private val CORRESPONDENCE = LinearCorrespondence(16f) | ||
| } | ||
|
|
||
| override fun toFlow(scope: CoroutineScope) = flow { | ||
| this@HueHistogramRetriever.reader.queryAndJoin(this@HueHistogramRetriever.query).forEach { | ||
| val distance = it.filteredAttribute<DistanceAttribute>() | ||
| if (distance != null) { | ||
| it.addAttribute(CORRESPONDENCE(distance)) | ||
| } else { | ||
| this@HueHistogramRetriever.logger.warn { "No distance attribute found for descriptor ${it.id}." } | ||
| } | ||
| emit(it) | ||
| } | ||
| } | ||
| } |
1 change: 1 addition & 0 deletions
1
...res/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,3 +1,4 @@ | ||
| org.vitrivr.engine.module.features.feature.ehd.EHD | ||
| org.vitrivr.engine.module.features.feature.huehistogram.HueHistogram | ||
| org.vitrivr.engine.module.features.feature.external.implementations.dino.DINO | ||
| org.vitrivr.engine.module.features.feature.external.implementations.clip.CLIP |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters