From 8765618bd7a31f1dd6b4431df9720b6a6202ed78 Mon Sep 17 00:00:00 2001 From: Ralph Gasser Date: Thu, 28 Dec 2023 11:05:21 +0100 Subject: [PATCH] ContentElements are now all associated with a ContentType, which is more stable to distinguish elements than doing so based on a class using reflection. Signed-off-by: Ralph Gasser --- .../engine/core/model/content/ContentType.kt | 21 +++++++++++++++++++ .../model/content/element/AudioContent.kt | 5 +++++ .../model/content/element/ContentElement.kt | 4 ++++ .../model/content/element/ImageContent.kt | 5 +++++ .../core/model/content/element/TextContent.kt | 6 ++++++ .../aggregators/CenterContentAggregator.kt | 4 +--- .../aggregators/FirstContentAggregator.kt | 2 +- .../aggregators/LastContentAggregator.kt | 2 +- .../transform/ContentSamplingTransformer.kt | 18 ++++++++-------- 9 files changed, 53 insertions(+), 14 deletions(-) create mode 100644 vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/ContentType.kt diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/ContentType.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/ContentType.kt new file mode 100644 index 00000000..0d1ec96c --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/ContentType.kt @@ -0,0 +1,21 @@ +package org.vitrivr.engine.core.model.content + +/** + * An enumeration of the type of content that can be processed by vitrivr-engine. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +enum class ContentType { + /** A bitmap image (e.g., extracted from a video or image file). */ + BITMAP_IMAGE, + + /** An audio frame (e.g., extracted from a video or audio file). */ + AUDIO_FRAME, + + /** Text (e.g., extracted from a document). */ + TEXT, + + /** A 3D mesh. */ + MESH +} \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/element/AudioContent.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/element/AudioContent.kt index 85052f7e..63277489 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/element/AudioContent.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/element/AudioContent.kt @@ -1,5 +1,6 @@ package org.vitrivr.engine.core.model.content.element +import org.vitrivr.engine.core.model.content.ContentType import java.nio.ShortBuffer /** @@ -19,4 +20,8 @@ interface AudioContent: ContentElement { /** The sampling rate of the data encoded in this [AudioContent]. */ val samplingRate: Int + + /** The [ContentType] of an [AudioContent] is always [ContentType.AUDIO_FRAME]. */ + override val type: ContentType + get() = ContentType.AUDIO_FRAME } \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/element/ContentElement.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/element/ContentElement.kt index a943fc21..ef5de14a 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/element/ContentElement.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/element/ContentElement.kt @@ -1,6 +1,7 @@ package org.vitrivr.engine.core.model.content.element import org.vitrivr.engine.core.model.content.Content +import org.vitrivr.engine.core.model.content.ContentType /** * A [Content] element is a piece of [Content] that is tied to some actual [Content]. @@ -17,4 +18,7 @@ sealed interface ContentElement: Content { * @return [ContentElement] */ val content: T + + /** The [ContentType] of this [ContentElement]. */ + val type: ContentType } \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/element/ImageContent.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/element/ImageContent.kt index 5e6dc3c3..1c5c2183 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/element/ImageContent.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/element/ImageContent.kt @@ -1,5 +1,6 @@ package org.vitrivr.engine.core.model.content.element +import org.vitrivr.engine.core.model.content.ContentType import java.awt.image.BufferedImage /** @@ -17,4 +18,8 @@ interface ImageContent: ContentElement { /** Height of the [BufferedImage] held by this [ContentElement]. */ val height: Int get() = this.content.height + + /** The [ContentType] of an [ImageContent] is always [ContentType.BITMAP_IMAGE]. */ + override val type: ContentType + get() = ContentType.BITMAP_IMAGE } \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/element/TextContent.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/element/TextContent.kt index 6d0ce53f..488e8af3 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/element/TextContent.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/element/TextContent.kt @@ -1,5 +1,7 @@ package org.vitrivr.engine.core.model.content.element +import org.vitrivr.engine.core.model.content.ContentType + /** * A textual [ContentElement]. * @@ -11,4 +13,8 @@ interface TextContent: ContentElement { /** Length of the [String] held by this [TextContent]. */ val length: Int get() = this.content.length + + /** The [ContentType] of an [TextContent] is always [ContentType.TEXT]. */ + override val type: ContentType + get() = ContentType.TEXT } \ No newline at end of file diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/CenterContentAggregator.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/CenterContentAggregator.kt index f2d3457c..2730c80e 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/CenterContentAggregator.kt +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/CenterContentAggregator.kt @@ -30,9 +30,7 @@ class CenterContentAggregator : AggregatorFactory { * The [Instance] returns by the [AggregatorFactory] */ private class Instance(override val input: Operator, context: IndexContext) : AbstractAggregator(input, context) { - override fun aggregate(content: List>): List> = content.groupBy { - it::class - }.mapNotNull { (_, elements) -> + override fun aggregate(content: List>): List> = content.groupBy { it.type }.mapNotNull { (_, elements) -> if (elements.isNotEmpty()) { elements[Math.floorDiv(elements.size, 2)] } else { diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/FirstContentAggregator.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/FirstContentAggregator.kt index 5843d840..0f283a7f 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/FirstContentAggregator.kt +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/FirstContentAggregator.kt @@ -30,6 +30,6 @@ class FirstContentAggregator : AggregatorFactory { * The [Instance] returns by the [AggregatorFactory] */ private class Instance(override val input: Operator, context: IndexContext) : AbstractAggregator(input, context) { - override fun aggregate(content: List>): List> = content.groupBy { it::class }.mapNotNull { (_, elements) -> elements.firstOrNull() } + override fun aggregate(content: List>): List> = content.groupBy { it.type }.mapNotNull { (_, elements) -> elements.firstOrNull() } } } \ No newline at end of file diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/LastContentAggregator.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/LastContentAggregator.kt index 6c361eb7..e7db4bbb 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/LastContentAggregator.kt +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/LastContentAggregator.kt @@ -30,6 +30,6 @@ class LastContentAggregator : AggregatorFactory { * The [Instance] returns by the [AggregatorFactory] */ private class Instance(override val input: Operator, context: IndexContext) : AbstractAggregator(input, context) { - override fun aggregate(content: List>): List> = content.groupBy { it::class }.mapNotNull { (_, elements) -> elements.lastOrNull() } + override fun aggregate(content: List>): List> = content.groupBy { it.type }.mapNotNull { (_, elements) -> elements.lastOrNull() } } } \ No newline at end of file diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/transform/ContentSamplingTransformer.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/transform/ContentSamplingTransformer.kt index eaf9146a..e70f0969 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/transform/ContentSamplingTransformer.kt +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/transform/ContentSamplingTransformer.kt @@ -4,6 +4,7 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.filter import org.vitrivr.engine.core.context.IndexContext +import org.vitrivr.engine.core.model.content.ContentType import org.vitrivr.engine.core.model.content.decorators.SourcedContent import org.vitrivr.engine.core.model.content.element.ContentElement import org.vitrivr.engine.core.operators.Operator @@ -11,7 +12,6 @@ import org.vitrivr.engine.core.operators.ingest.Decoder import org.vitrivr.engine.core.operators.ingest.Transformer import org.vitrivr.engine.core.operators.ingest.TransformerFactory import org.vitrivr.engine.core.source.Source -import kotlin.reflect.KClass /** * A [Transformer] that samples the input [Flow] and only passes through every n-th element. @@ -25,17 +25,17 @@ class ContentSamplingTransformer : TransformerFactory { private class Instance(override val input: Operator>, private val sample: Int) : Transformer { override fun toFlow(scope: CoroutineScope): Flow> { - val sources = mutableMapOf>, Source>() - val counters = mutableMapOf>, Int>() + val sources = mutableMapOf() + val counters = mutableMapOf() return this.input.toFlow(scope).filter { value: ContentElement<*> -> if (value is SourcedContent) { /* Only source content can be sampled. */ - val clazz = value::class - if (sources[clazz] == null || sources[clazz] != value.source) { - sources[clazz] = value.source - counters[clazz] = 0 + val type = value.type + if (sources[type] == null || sources[type] != value.source) { + sources[type] = value.source + counters[type] = 0 } - val pass = (counters[clazz]!! % this.sample == 0) - counters[clazz] = counters[clazz]!! + 1 + val pass = (counters[type]!! % this.sample == 0) + counters[type] = counters[type]!! + 1 pass } else { true