made sure to actually filter out sources

vitrivr · Aug 26, 2024 · 4fe063c · 4fe063c
1 parent 91fe2f1
commit 4fe063c
Show file tree

Hide file tree

Showing 12 changed files with 23 additions and 18 deletions.
diff --git a/...-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/AbstractBatchedExtractor.kt b/...-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/AbstractBatchedExtractor.kt
@@ -41,7 +41,7 @@ abstract class AbstractBatchedExtractor<C : ContentElement<*>, D : Descriptor<*>
             retrievable.filteredAttribute(ContentAuthorAttribute::class.java)?.getContentIds(it)
         }
         return retrievable.content.filter { content ->
-            this.analyser.contentClasses.any { it.isInstance(content) && (contentIds?.contains(content.id) ?: false) }
+        this.analyser.contentClasses.any { it.isInstance(content) && (contentIds?.contains(content.id) ?: true) }
         }.map { it as C }
     }
 

diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/AbstractExtractor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/AbstractExtractor.kt
@@ -37,8 +37,8 @@ abstract class AbstractExtractor<C : ContentElement<*>, D : Descriptor<*>>(final
             retrievable.filteredAttribute(ContentAuthorAttribute::class.java)?.getContentIds(it)
         }
         return retrievable.content.filter { content ->
-            this.analyser.contentClasses.any { it.isInstance(content) && (contentIds?.contains(content.id) ?: false) }
-        }.map { it as C }
+            this.analyser.contentClasses.any { it.isInstance(content) && (contentIds?.contains(content.id) ?: true) }
+        }.map { it as C}
     }
 
     /**

diff --git a/...re/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColorExtractor.kt b/...re/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColorExtractor.kt
@@ -41,7 +41,7 @@ class AverageColorExtractor(input: Operator<Retrievable>, analyser: AverageColor
      * @return List of resulting [Descriptor]s.
      */
     override fun extract(retrievable: Retrievable): List<FloatVectorDescriptor> {
-        val content = retrievable.content.filterIsInstance<ImageContent>()
+        val content = this.filterContent(retrievable)
         return content.map { (this.analyser as AverageColor).analyse(it).copy(retrievableId = retrievable.id, field = this.field) }
     }
 }
diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/exporters/ThumbnailExporter.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/exporters/ThumbnailExporter.kt
@@ -11,6 +11,7 @@ import kotlinx.coroutines.flow.onEach
 import org.vitrivr.engine.core.context.IndexContext
 import org.vitrivr.engine.core.model.content.element.ImageContent
 import org.vitrivr.engine.core.model.retrievable.Retrievable
+import org.vitrivr.engine.core.model.retrievable.attributes.ContentAuthorAttribute
 import org.vitrivr.engine.core.operators.Operator
 import org.vitrivr.engine.core.operators.general.Exporter
 import org.vitrivr.engine.core.operators.general.ExporterFactory
@@ -46,20 +47,24 @@ class ThumbnailExporter : ExporterFactory {
             }
         } ?: MimeType.JPG
         logger.debug { "Creating new ThumbnailExporter with maxSideResolution=$maxSideResolution and mimeType=$mimeType." }
-        return Instance(input, context, maxSideResolution, mimeType)
+        return Instance(input, context, maxSideResolution, mimeType, context[name, "contentSources"]?.split(",")?.toSet() ?: emptySet())
     }
 
     /**
      * The [Exporter] generated by this [ThumbnailExporter].
      */
-    private class Instance(override val input: Operator<Retrievable>, private val context: IndexContext, private val maxResolution: Int, private val mimeType: MimeType) : Exporter {
+    private class Instance(override val input: Operator<Retrievable>, private val context: IndexContext, private val maxResolution: Int, private val mimeType: MimeType, private val contentSources:Set<String>) : Exporter {
         init {
             require(mimeType in SUPPORTED) { "ThumbnailExporter only support image formats JPEG and PNG." }
         }
 
+
         override fun toFlow(scope: CoroutineScope): Flow<Retrievable> = this.input.toFlow(scope).onEach { retrievable ->
             val resolvable = this.context.resolver.resolve(retrievable.id)
-            val content = retrievable.content.filterIsInstance<ImageContent>()
+
+            val contentIds = retrievable.filteredAttribute(ContentAuthorAttribute::class.java)?.getContentIds(contentSources)
+
+            val content = retrievable.content.filterIsInstance<ImageContent>().filter { contentIds?.contains(it.id) ?: true }
             if (resolvable != null && content.isNotEmpty()) {
                 val writer = when (mimeType) {
                     MimeType.JPEG,

diff --git a/...e-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/cld/CLDExtractor.kt b/...e-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/cld/CLDExtractor.kt
@@ -38,7 +38,7 @@ class CLDExtractor(input: Operator<Retrievable>, analyser: CLD, field: Schema.Fi
      * @return List of resulting [Descriptor]s.
      */
     override fun extract(retrievable: Retrievable): List<FloatVectorDescriptor> {
-        val content = retrievable.content.filterIsInstance<ImageContent>()
+        val content = this.filterContent(retrievable)
         return content.map { (this.analyser as CLD).analyse(it).copy(retrievableId = retrievable.id, field = this.field) }
     }
 }
diff --git a/...kotlin/org/vitrivr/engine/module/features/feature/dominantcolor/DominantColorExtractor.kt b/...kotlin/org/vitrivr/engine/module/features/feature/dominantcolor/DominantColorExtractor.kt
@@ -26,7 +26,7 @@ class DominantColorExtractor(input: Operator<Retrievable>, analyser: DominantCol
 
 
     override fun extract(retrievable: Retrievable): List<LabelDescriptor> {
-        val content = retrievable.content.filterIsInstance<ImageContent>()
+        val content = this.filterContent(retrievable)
         return (this.analyser as DominantColor).analyse(content).map { it.copy(retrievableId = retrievable.id, field = this@DominantColorExtractor.field) }
     }
 

diff --git a/...e-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/ehd/EHDExtractor.kt b/...e-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/ehd/EHDExtractor.kt
@@ -38,7 +38,7 @@ class EHDExtractor(input: Operator<Retrievable>, analyser: EHD, field: Schema.Fi
      * @return List of resulting [Descriptor]s.
      */
     override fun extract(retrievable: Retrievable): List<FloatVectorDescriptor> {
-        val content = retrievable.content.filterIsInstance<ImageContent>()
+        val content = this.filterContent(retrievable)
         return content.map { (this.analyser as EHD).analyse(it).copy(retrievableId = retrievable.id, field = this.field) }
     }
 }
diff --git a/...org/vitrivr/engine/module/features/feature/external/implementations/clip/CLIPExtractor.kt b/...org/vitrivr/engine/module/features/feature/external/implementations/clip/CLIPExtractor.kt
@@ -36,7 +36,7 @@ class CLIPExtractor(input: Operator<Retrievable>, analyser: CLIP, field: Schema.
      * @return List of resulting [Descriptor]s.
      */
     override fun extract(retrievable: Retrievable): List<FloatVectorDescriptor> {
-        val content = retrievable.content.filterIsInstance<ImageContent>()
+        val content = this.filterContent(retrievable)
         return content.map { c ->
             CLIP.analyse(c, this.host).copy(retrievableId = retrievable.id, field = this@CLIPExtractor.field)
         }

diff --git a/...org/vitrivr/engine/module/features/feature/external/implementations/dino/DINOExtractor.kt b/...org/vitrivr/engine/module/features/feature/external/implementations/dino/DINOExtractor.kt
@@ -36,7 +36,7 @@ class DINOExtractor(input: Operator<Retrievable>, analyser: DINO, field: Schema.
      * @return List of resulting [Descriptor]s.
      */
     override fun extract(retrievable: Retrievable): List<FloatVectorDescriptor> {
-        val content = retrievable.content.filterIsInstance<ImageContent>()
+        val content = this.filterContent(retrievable)
         return content.map { c ->
             DINO.analyse(c, this.host).copy(retrievableId = retrievable.id, field = this@DINOExtractor.field)
         }

diff --git a/...n/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogramExtractor.kt b/...n/kotlin/org/vitrivr/engine/module/features/feature/huehistogram/HueHistogramExtractor.kt
@@ -39,7 +39,7 @@ class HueHistogramExtractor(input: Operator<Retrievable>, analyser: HueHistogram
      * @return List of resulting [Descriptor]s.
      */
     override fun extract(retrievable: Retrievable): List<FloatVectorDescriptor> {
-        val content = retrievable.content.filterIsInstance<ImageContent>()
+        val content = this.filterContent(retrievable)
         return content.map { (this.analyser as HueHistogram).analyse(it).copy(retrievableId = retrievable.id, field = this.field) }
     }
 }
diff --git a/...ain/kotlin/org/vitrivr/engine/module/features/feature/mediancolor/MedianColorExtractor.kt b/...ain/kotlin/org/vitrivr/engine/module/features/feature/mediancolor/MedianColorExtractor.kt
@@ -39,7 +39,7 @@ class MedianColorExtractor(input: Operator<Retrievable>, analyser: MedianColor,
      * @return List of resulting [Descriptor]s.
      */
     override fun extract(retrievable: Retrievable): List<FloatVectorDescriptor> {
-        val content = retrievable.content.filterIsInstance<ImageContent>()
+        val content = this.filterContent(retrievable)
         return content.map { (this.analyser as MedianColor).analyse(it).copy(retrievableId = retrievable.id, field = this.field) }
     }
 }
diff --git a/...lin/org/vitrivr/engine/model3d/features/sphericalharmonics/SphericalHarmonicsExtractor.kt b/...lin/org/vitrivr/engine/model3d/features/sphericalharmonics/SphericalHarmonicsExtractor.kt
@@ -27,9 +27,9 @@ class SphericalHarmonicsExtractor(
     private val gridSize: Int,
     private val cap: Int,
     private val minL: Int,
-    private val maxL: Int
-) :
-    AbstractExtractor<Model3DContent, FloatVectorDescriptor>(input, analyser, field) {
+    private val maxL: Int,
+    parameters : Map<String,String>
+) : AbstractExtractor<Model3DContent, FloatVectorDescriptor>(input, analyser, field, parameters) {
 
     init {
         require(this.minL < this.maxL) { "Parameter mismatch: min_l must be smaller than max_l. "}
@@ -52,7 +52,7 @@ class SphericalHarmonicsExtractor(
      * @return List of resulting [FloatVectorDescriptor]s.
      */
     override fun extract(retrievable: Retrievable): List<FloatVectorDescriptor> {
-        val content = retrievable.content.filterIsInstance<Model3DContent>()
+        val content = this.filterContent(retrievable)
         return content.flatMap { c -> c.content.getMaterials().flatMap { mat -> mat.meshes.map { mesh -> SphericalHarmonics.analyse(mesh, this.gridSize, this.minL, this.maxL, this.cap).copy(field = this.field) } } }
     }
 }