Merge of dev.

Signed-off-by: Ralph Gasser <[email protected]>

settings.gradle

@@ -13,4 +13,3 @@ include 'vitrivr-engine-module-pgvector'
 include 'vitrivr-engine-module-features'
 include 'vitrivr-engine-module-m3d'
 include 'vitrivr-engine-module-fes'
-include 'vitrivr-engine-module-statistics'

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/pipeline/execution/RetrievalPipeline.kt

@@ -6,8 +6,6 @@ import org.vitrivr.engine.core.operators.Operator
 /**
  * A pipeline for retrieval. It wraps a (query) [Operator] that returns [Retrieved] objects.
  *
- * TODO: Builder for this must be implemented.
- *
  * @author Ralph Gasser
  * @version 1.0.0
  */

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/context/Context.kt

@@ -2,7 +2,6 @@ package org.vitrivr.engine.core.context
 
 import io.javalin.openapi.OpenApiIgnore
 import kotlinx.serialization.Serializable
-import kotlinx.serialization.Transient
 import org.vitrivr.engine.core.model.metamodel.Schema
 
 /**
@@ -11,7 +10,7 @@ import org.vitrivr.engine.core.model.metamodel.Schema
  * This is applicable for index and query phases.
  */
 @Serializable
-sealed class Context() {
+sealed class Context {
     /**
      * Configuration per named operator.
      */

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/BlackholeConnection.kt

@@ -0,0 +1,68 @@
+package org.vitrivr.engine.core.database.blackhole
+
+import io.github.oshai.kotlinlogging.KLogger
+import io.github.oshai.kotlinlogging.KotlinLogging.logger
+import io.github.oshai.kotlinlogging.Marker
+import org.vitrivr.engine.core.database.AbstractConnection
+import org.vitrivr.engine.core.database.ConnectionProvider
+import org.vitrivr.engine.core.database.blackhole.descriptors.BlackholeDescriptorInitializer
+import org.vitrivr.engine.core.database.blackhole.descriptors.BlackholeDescriptorReader
+import org.vitrivr.engine.core.database.blackhole.descriptors.BlackholeDescriptorWriter
+import org.vitrivr.engine.core.database.blackhole.retrievable.BlackholeRetrievableInitializer
+import org.vitrivr.engine.core.database.blackhole.retrievable.BlackholeRetrievableReader
+import org.vitrivr.engine.core.database.blackhole.retrievable.BlackholeRetrievableWriter
+import org.vitrivr.engine.core.database.descriptor.DescriptorReader
+import org.vitrivr.engine.core.database.retrievable.RetrievableInitializer
+import org.vitrivr.engine.core.database.retrievable.RetrievableReader
+import org.vitrivr.engine.core.database.retrievable.RetrievableWriter
+import org.vitrivr.engine.core.model.descriptor.Descriptor
+import org.vitrivr.engine.core.model.metamodel.Schema
+
+
+/** Defines [KLogger] of the class. */
+internal val LOGGER: KLogger = logger {}
+
+/**
+ * An [AbstractConnection] that swallows all data and does not store anything. However, it can be used to log operations.
+ *
+ * @author Ralph Gasser
+ * @version 1.0.0
+ */
+class BlackholeConnection(schemaName: String, provider: ConnectionProvider, private val log: Boolean = false) : AbstractConnection(schemaName, provider) {
+
+
+
+    init {
+        LOGGER.warn { "You are using the blackhole connection with schema $schemaName. No data will be stored!" }
+    }
+
+    /** [Marker] used for logging. */
+    private val marker: Marker = object : Marker {
+        override fun getName(): String = this@BlackholeConnection.description()
+    }
+
+    override fun initialize() = this.logIf("Initializing schema '$schemaName'.")
+    override fun truncate() = this.logIf("Truncating schema '$schemaName'.")
+    override fun <T> withTransaction(action: (Unit) -> T): T {
+        LOGGER.warn { "Transactions are not supported by blackhole connection. Ignoring transaction." }
+        return action(Unit)
+    }
+
+    override fun getRetrievableInitializer(): RetrievableInitializer = BlackholeRetrievableInitializer(this)
+    override fun getRetrievableWriter(): RetrievableWriter = BlackholeRetrievableWriter(this)
+    override fun getRetrievableReader(): RetrievableReader = BlackholeRetrievableReader(this)
+    override fun <D : Descriptor<*>> getDescriptorInitializer(field: Schema.Field<*, D>) = BlackholeDescriptorInitializer(this, field)
+    override fun <D : Descriptor<*>> getDescriptorWriter(field: Schema.Field<*, D>) = BlackholeDescriptorWriter(this, field)
+    override fun <D : Descriptor<*>> getDescriptorReader(field: Schema.Field<*, D>): DescriptorReader<D> = BlackholeDescriptorReader(this, field)
+    override fun description(): String = "'$schemaName'@blackhole"
+    override fun close() = this.logIf("Closing connection to blackhole database.")
+
+    /**
+     * Conditionally logs a message if logging is enabled.
+     *
+     * @param message The message to log.
+     */
+    internal fun logIf(message: String) {
+        if (this.log) LOGGER.info(throwable = null, marker = this.marker) { message }
+    }
+}

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/BlackholeConnectionProvider.kt

@@ -0,0 +1,23 @@
+package org.vitrivr.engine.core.database.blackhole
+
+import org.vitrivr.engine.core.database.ConnectionProvider
+import org.vitrivr.engine.core.database.blackhole.descriptors.BlackholeDescriptionProvider
+import org.vitrivr.engine.core.database.descriptor.DescriptorProvider
+import org.vitrivr.engine.core.model.descriptor.Descriptor
+import kotlin.reflect.KClass
+
+/**
+ * A [ConnectionProvider] for the [BlackholeConnection].
+ *
+ * @author Ralph Gasser
+ * @version 1.0.0
+ */
+class BlackholeConnectionProvider : ConnectionProvider {
+    override val databaseName: String = "blackhole"
+    override val version: String = "1.0.0"
+    override fun openConnection(schemaName: String, parameters: Map<String, String>) = BlackholeConnection(schemaName, this, parameters["log"]?.toBoolean() == true)
+    override fun <T : Descriptor<*>> register(descriptorClass: KClass<T>, provider: DescriptorProvider<*>) { /* No op. */
+    }
+
+    override fun <T : Descriptor<*>> obtain(descriptorClass: KClass<T>) = BlackholeDescriptionProvider<T>()
+}

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/descriptors/BlackholeDescriptionProvider.kt

@@ -0,0 +1,21 @@
+package org.vitrivr.engine.core.database.blackhole.descriptors
+
+import org.vitrivr.engine.core.database.Connection
+import org.vitrivr.engine.core.database.blackhole.BlackholeConnection
+import org.vitrivr.engine.core.database.descriptor.DescriptorProvider
+import org.vitrivr.engine.core.database.descriptor.DescriptorReader
+import org.vitrivr.engine.core.database.descriptor.DescriptorWriter
+import org.vitrivr.engine.core.model.descriptor.Descriptor
+import org.vitrivr.engine.core.model.metamodel.Schema
+
+/**
+ * A [DescriptorProvider] for the [BlackholeConnection].
+ *
+ * @author Ralph Gasser
+ * @version 1.0.0
+ */
+class BlackholeDescriptionProvider<T : Descriptor<*>> : DescriptorProvider<T> {
+    override fun newInitializer(connection: Connection, field: Schema.Field<*, T>) = BlackholeDescriptorInitializer(connection as BlackholeConnection, field)
+    override fun newReader(connection: Connection, field: Schema.Field<*, T>): DescriptorReader<T> = BlackholeDescriptorReader(connection as BlackholeConnection, field)
+    override fun newWriter(connection: Connection, field: Schema.Field<*, T>): DescriptorWriter<T> = BlackholeDescriptorWriter(connection as BlackholeConnection, field)
+}

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/descriptors/BlackholeDescriptorInitializer.kt

@@ -0,0 +1,19 @@
+package org.vitrivr.engine.core.database.blackhole.descriptors
+
+import org.vitrivr.engine.core.database.blackhole.BlackholeConnection
+import org.vitrivr.engine.core.database.descriptor.DescriptorInitializer
+import org.vitrivr.engine.core.model.descriptor.Descriptor
+import org.vitrivr.engine.core.model.metamodel.Schema
+
+/**
+ * A [DescriptorInitializer] for the [BlackholeConnection].
+ *
+ * @author Ralph Gasser
+ * @version 1.0.0
+ */
+class BlackholeDescriptorInitializer<T : Descriptor<*>>(private val connection: BlackholeConnection, override val field: Schema.Field<*, T>) : DescriptorInitializer<T> {
+    override fun initialize() = this.connection.logIf("Initializing descriptor entity '${this.field.fieldName}'.")
+    override fun deinitialize() = this.connection.logIf("De-initializing descriptor entity '${this.field.fieldName}'.")
+    override fun isInitialized(): Boolean = false
+    override fun truncate() = this.connection.logIf("Truncating descriptor entity '${this.field.fieldName}'.")
+}

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/descriptors/BlackholeDescriptorReader.kt

@@ -0,0 +1,28 @@
+package org.vitrivr.engine.core.database.blackhole.descriptors
+
+import org.vitrivr.engine.core.database.blackhole.BlackholeConnection
+import org.vitrivr.engine.core.database.descriptor.DescriptorReader
+import org.vitrivr.engine.core.model.descriptor.Descriptor
+import org.vitrivr.engine.core.model.descriptor.DescriptorId
+import org.vitrivr.engine.core.model.metamodel.Schema
+import org.vitrivr.engine.core.model.query.Query
+import org.vitrivr.engine.core.model.retrievable.RetrievableId
+import org.vitrivr.engine.core.model.retrievable.Retrieved
+
+/**
+ * A [DescriptorReader] for the [BlackholeConnection].
+ *
+ * @author Ralph Gasser
+ * @version 1.0.0
+ */
+class BlackholeDescriptorReader<T : Descriptor<*>>(override val connection: BlackholeConnection, override val field: Schema.Field<*, T>) : DescriptorReader<T> {
+    override fun exists(descriptorId: DescriptorId): Boolean = false
+    override fun get(descriptorId: DescriptorId): T? = null
+    override fun getAll(descriptorIds: Iterable<DescriptorId>) = emptySequence<T>()
+    override fun getAll() = emptySequence<T>()
+    override fun getForRetrievable(retrievableId: RetrievableId) = emptySequence<T>()
+    override fun getAllForRetrievable(retrievableIds: Iterable<RetrievableId>) = emptySequence<T>()
+    override fun query(query: Query) = emptySequence<T>()
+    override fun queryAndJoin(query: Query) = emptySequence<Retrieved>()
+    override fun count() = 0L
+}

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/descriptors/BlackholeDescriptorWriter.kt

@@ -0,0 +1,39 @@
+package org.vitrivr.engine.core.database.blackhole.descriptors
+
+import org.vitrivr.engine.core.database.blackhole.BlackholeConnection
+import org.vitrivr.engine.core.database.descriptor.DescriptorWriter
+import org.vitrivr.engine.core.model.descriptor.Descriptor
+import org.vitrivr.engine.core.model.metamodel.Schema
+
+/**
+ * A [DescriptorWriter] for the [BlackholeConnection].
+ *
+ * @author Ralph Gasser
+ * @version 1.0.0
+ */
+class BlackholeDescriptorWriter<T : Descriptor<*>>(override val connection: BlackholeConnection, override val field: Schema.Field<*, T>) : DescriptorWriter<T> {
+    override fun add(item: T): Boolean {
+        this.connection.logIf("Adding descriptor '${item.id}' to entity '${this.field.fieldName}'.")
+        return false
+    }
+
+    override fun addAll(items: Iterable<T>): Boolean {
+        items.forEach { item -> this.connection.logIf("Adding descriptor '${item.id}' to entity '${this.field.fieldName}'.") }
+        return false
+    }
+
+    override fun update(item: T): Boolean {
+        this.connection.logIf("Updating descriptor '${item.id}' in entity '${this.field.fieldName}'.")
+        return false
+    }
+
+    override fun delete(item: T): Boolean {
+        this.connection.logIf("Deleting descriptor '${item.id}' in entity '${this.field.fieldName}'.")
+        return false
+    }
+
+    override fun deleteAll(items: Iterable<T>): Boolean {
+        items.forEach { item -> this.connection.logIf("Deleting descriptor '${item.id}' from entity '${this.field.fieldName}'.") }
+        return false
+    }
+}

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/retrievable/BlackholeRetrievableInitializer.kt

@@ -0,0 +1,17 @@
+package org.vitrivr.engine.core.database.blackhole.retrievable
+
+import org.vitrivr.engine.core.database.blackhole.BlackholeConnection
+import org.vitrivr.engine.core.database.retrievable.RetrievableInitializer
+
+/**
+ * A [RetrievableInitializer] for the [BlackholeConnection].
+ *
+ * @author Ralph Gasser
+ * @version 1.0.0
+ */
+class BlackholeRetrievableInitializer(private val connection: BlackholeConnection) : RetrievableInitializer {
+    override fun initialize() = this.connection.logIf("Initializing entities 'retrievable' and 'relationship'.")
+    override fun deinitialize() = this.connection.logIf("De-initializing entities 'retrievable' and 'relationship'.")
+    override fun isInitialized(): Boolean = false
+    override fun truncate() = this.connection.logIf("Truncating entities 'retrievable' and 'relationship'.")
+}

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/retrievable/BlackholeRetrievableReader.kt

@@ -0,0 +1,22 @@
+package org.vitrivr.engine.core.database.blackhole.retrievable
+
+import org.vitrivr.engine.core.database.blackhole.BlackholeConnection
+import org.vitrivr.engine.core.database.retrievable.RetrievableReader
+import org.vitrivr.engine.core.model.retrievable.Retrievable
+import org.vitrivr.engine.core.model.retrievable.RetrievableId
+
+
+/**
+ * A [RetrievableReader] for the [BlackholeConnection].
+ *
+ * @author Ralph Gasser
+ * @version 1.0.0
+ */
+class BlackholeRetrievableReader(override val connection: BlackholeConnection) : RetrievableReader {
+    override fun get(id: RetrievableId): Retrievable? = null
+    override fun exists(id: RetrievableId): Boolean = false
+    override fun getAll(ids: Iterable<RetrievableId>) = emptySequence<Retrievable>()
+    override fun getAll() = emptySequence<Retrievable>()
+    override fun getConnections(subjectIds: Collection<RetrievableId>, predicates: Collection<String>, objectIds: Collection<RetrievableId>) = emptySequence<Triple<RetrievableId, String, RetrievableId>>()
+    override fun count(): Long = 0L
+}

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/retrievable/BlackholeRetrievableWriter.kt

@@ -0,0 +1,60 @@
+package org.vitrivr.engine.core.database.blackhole.retrievable
+
+import org.vitrivr.engine.core.database.blackhole.BlackholeConnection
+import org.vitrivr.engine.core.database.retrievable.RetrievableWriter
+import org.vitrivr.engine.core.model.relationship.Relationship
+import org.vitrivr.engine.core.model.retrievable.Retrievable
+
+
+/**
+ * A [RetrievableWriter] for the [BlackholeConnection].
+ *
+ * @author Ralph Gasser
+ * @version 1.0.0
+ */
+class BlackholeRetrievableWriter(override val connection: BlackholeConnection) : RetrievableWriter {
+    override fun connect(relationship: Relationship): Boolean {
+        this.connection.logIf("Adding relationship ${relationship.subjectId} >[${relationship.predicate}]  ${relationship.objectId}.")
+        return false
+    }
+
+    override fun connectAll(relationships: Iterable<Relationship>): Boolean {
+        relationships.forEach { relationship -> this.connection.logIf("Adding relationship ${relationship.subjectId} >[${relationship.predicate}]  ${relationship.objectId}.") }
+        return false
+    }
+
+    override fun disconnect(relationship: Relationship): Boolean {
+        this.connection.logIf("Removing relationship ${relationship.subjectId} >[${relationship.predicate}]  ${relationship.objectId}.")
+        return false
+    }
+
+    override fun disconnectAll(relationships: Iterable<Relationship>): Boolean {
+        relationships.forEach { relationship -> this.connection.logIf("Removing relationship ${relationship.subjectId} >[${relationship.predicate}]  ${relationship.objectId}.") }
+        return false
+    }
+
+    override fun add(item: Retrievable): Boolean {
+        this.connection.logIf("Adding retrievable '${item.id}' to entity 'retrievable'.")
+        return false
+    }
+
+    override fun addAll(items: Iterable<Retrievable>): Boolean {
+        items.forEach { item -> this.connection.logIf("Adding retrievable '${item.id}' to entity 'retrievable'.") }
+        return false
+    }
+
+    override fun update(item: Retrievable): Boolean {
+        this.connection.logIf("Updating retrievable '${item.id}' in entity 'retrievable'.")
+        return false
+    }
+
+    override fun delete(item: Retrievable): Boolean {
+        this.connection.logIf("Deleting retrievable '${item.id}' from entity 'retrievable'.")
+        return false
+    }
+
+    override fun deleteAll(items: Iterable<Retrievable>): Boolean {
+        items.forEach { item -> this.connection.logIf("Deleting retrievable '${item.id}' from entity 'retrievable'.") }
+        return false
+    }
+}

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/metadata/source/video/VideoSourceMetadataExtractor.kt

@@ -49,6 +49,7 @@ class VideoSourceMetadataExtractor(input: Operator<Retrievable>, analyser: Video
                 mapOf(
                     "width" to Value.Int(source.width() ?: 0),
                     "height" to Value.Int(source.height() ?: 0),
+                    "duration" to Value.Long(source.duration() ?: 0L),
                     "fps" to Value.Double(source.fps() ?: 0.0),
                     "channels" to Value.Int(source.channels() ?: 0),
                     "sampleRate" to Value.Int(source.sampleRate() ?: 0),