merge with dev

.github/workflows/ci.yml

@@ -0,0 +1,50 @@
+name: Java CI with Gradle
+
+on:
+  push:
+    branches: [ main, dev ]
+  pull_request:
+    branches: [ main, dev ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    # Setup Cottontail DB and PostgreSQL service container
+    services:
+      cottontail:
+        image: vitrivr/cottontaildb:0.16.7
+        ports:
+          - 1865:1865
+        options: -it
+      pgvector:
+        image: pgvector/pgvector:pg16
+        ports:
+          - 5432:5432
+        env:
+          POSTGRES_PASSWORD: vitrivr
+
+    # Start actual job.
+    steps:
+      - uses: actions/checkout@v4
+      - name: Cottontail DB connection test
+        run: nc -zv 127.0.0.1 1865
+      - name: PostgreSQL connection test
+        run: nc -zv 127.0.0.1 5432
+      - name: Set up JDK
+        uses: actions/setup-java@v4
+        with:
+          java-version: '21'
+          distribution: 'temurin'
+      - name: Cache Gradle packages
+        uses: actions/cache@v4
+        with:
+          path: ~/.gradle/caches
+          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
+          restore-keys: ${{ runner.os }}-gradle
+      - name: Test with gradle ubuntu
+        if: matrix.os == 'ubuntu-latest'
+        run: ./gradlew test --info
+      - name: Test with gradle windows
+        if: matrix.os == 'windows-latest'
+        run: ./gradlew test --info

build.gradle

@@ -47,9 +47,10 @@ subprojects {
     }
 
     /* Common plugins. */
-    apply plugin: 'java'
     apply plugin: 'idea'
     apply plugin: 'kotlin'
+    apply plugin: 'java'
+    apply plugin: 'java-test-fixtures'
 
     /* Java Version */
     sourceCompatibility = JavaVersion.VERSION_21

config-schema.json

@@ -5,7 +5,7 @@
       "connection": {
         "database": "CottontailConnectionProvider",
         "parameters": {
-          "Host": "127.0.0.1",
+          "host": "127.0.0.1",
           "port": "1865"
         }
       },

gradle.properties

@@ -7,6 +7,7 @@ version_javacv=1.5.9
 version_javalin=6.1.3
 version_javalinopenapi=6.1.3
 version_javalinssl=6.1.3
+version_jdbc_postgres=42.7.3
 version_jline=3.23.0
 version_junit=5.10.1
 version_junit_platform=1.10.1

settings.gradle

@@ -8,8 +8,8 @@ include 'vitrivr-engine-index'
 include 'vitrivr-engine-query'
 include 'vitrivr-engine-server'
 include 'vitrivr-engine-module-cottontaildb'
+include 'vitrivr-engine-module-pgvector'
 include 'vitrivr-engine-module-features'
 include 'vitrivr-engine-module-m3d'
 include 'vitrivr-engine-module-fes'
 include 'vitrivr-engine-module-statistics'
-

vitrivr-engine-core/build.gradle

@@ -7,10 +7,15 @@ plugins {
 dependencies {
     /** JOML dependencies for 3D mesh support. */
     implementation group: 'org.joml', name: 'joml', version: version_joml
+
     /** dependencies for exif metadata extraction. */
     implementation 'com.drewnoakes:metadata-extractor:2.19.0'
     implementation 'com.google.code.gson:gson:2.8.9'
     implementation group: 'io.javalin.community.openapi', name: 'javalin-openapi-plugin', version: version_javalinopenapi
+
+    /* Test Fixtures from Cottontail DB core. .*/
+    testFixturesImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-api', version: version_junit
+    testFixturesImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-params', version: version_junit
 }
 
 /* Publication of vitrivr engine core to Maven Central. */

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/schema/FieldConfig.kt

@@ -11,6 +11,5 @@ import org.vitrivr.engine.core.model.metamodel.Schema
  * @author Ralph Gasser
  * @version 1.0.0
  */
-
 @Serializable
-data class FieldConfig(val name: String, val factory: String, val parameters: Map<String,String> = emptyMap())
+data class FieldConfig(val name: String, val factory: String, val parameters: Map<String,String> = emptyMap(), val indexes: List<IndexConfig> = emptyList())

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/schema/IndexConfig.kt

@@ -0,0 +1,20 @@
+package org.vitrivr.engine.core.config.schema
+
+import kotlinx.serialization.Serializable
+import org.vitrivr.engine.core.model.descriptor.AttributeName
+import org.vitrivr.engine.core.model.metamodel.Schema
+
+/**
+ * A serializable configuration object to configure indexes for a [Schema.Field] within a [Schema].
+ *
+ * @see [Schema.Field]
+ *
+ * @author Ralph Gasser
+ * @version 1.0.0
+ */
+@Serializable
+data class IndexConfig(val attributes: List<AttributeName>, val type: IndexType, val parameters: Map<String, String> = emptyMap()) {
+    init {
+        require(attributes.isNotEmpty()) { "Cannot define index on empty list of attributes." }
+    }
+}

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/schema/IndexType.kt

@@ -0,0 +1,15 @@
+package org.vitrivr.engine.core.config.schema
+
+/**
+ * The type of index that can be created on a [Schema.Field].
+ */
+enum class IndexType {
+    /** A B-tree based index. Well-suited for point-lookups and Boolean search on scalar values. */
+    SCALAR,
+
+    /** A fulltext index. Well-suited for fulltext-queries. */
+    FULLTEXT,
+
+    /** Index suited for NNS. */
+    NNS
+}

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/schema/SchemaConfig.kt

@@ -1,9 +1,12 @@
 package org.vitrivr.engine.core.config.schema
 
 import kotlinx.serialization.Serializable
+import kotlinx.serialization.json.Json
 import org.vitrivr.engine.core.model.metamodel.Schema
 import org.vitrivr.engine.core.operators.general.Exporter
 import org.vitrivr.engine.core.resolver.Resolver
+import java.nio.file.Files
+import java.nio.file.Paths
 
 
 /**
@@ -43,4 +46,22 @@ data class SchemaConfig(
      * List of [PipelineConfig]s that are part of this [SchemaConfig].
      */
     val extractionPipelines: List<PipelineConfig> = emptyList()
-)
+) {
+
+    companion object {
+
+        /**
+         * Tries to load a [SchemaConfig] from the resources.
+         *
+         * @param resourcePath Path to the resource.
+         * @return [SchemaConfig]
+         */
+        fun loadFromResource(resourcePath: String): SchemaConfig {
+            val json = Json { ignoreUnknownKeys = true } // Configure Json to ignore unknown keys
+            val uri = this::class.java.classLoader.resources(resourcePath).findFirst().orElseThrow { IllegalArgumentException("Resource '$resourcePath' not found!") }.toURI()
+            val path = Paths.get(uri)
+            val jsonString = Files.readString(path)
+            return json.decodeFromString<SchemaConfig>(jsonString)
+        }
+    }
+}

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/AbstractConnectionProvider.kt

@@ -0,0 +1,50 @@
+package org.vitrivr.engine.core.database
+
+import org.vitrivr.engine.core.database.descriptor.DescriptorProvider
+import org.vitrivr.engine.core.model.descriptor.Descriptor
+import kotlin.reflect.KClass
+
+/**
+ * Abstract implementation of the [ConnectionProvider] interface, which provides basic facilities to register [DescriptorProvider]s.
+ *
+ * @author Ralph Gasser
+ * @version 1.0.0
+ */
+abstract class AbstractConnectionProvider: ConnectionProvider {
+    /** A map of registered [DescriptorProvider]. */
+    protected val registered: MutableMap<KClass<*>, DescriptorProvider<*>> = HashMap()
+
+
+    init {
+        this.initialize()
+    }
+
+    /**
+     * This method is called during initialization of the [AbstractConnectionProvider] and can be used to register [DescriptorProvider]s.
+     */
+    abstract fun initialize()
+
+    /**
+     * Registers an [DescriptorProvider] for a particular [KClass] of [Descriptor] with this [Connection].
+     *
+     * This method is an extension point to add support for new [Descriptor]s to a pre-existing database driver.
+     *
+     * @param descriptorClass The [KClass] of the [Descriptor] to register [DescriptorProvider] for.
+     * @param provider The [DescriptorProvider] to register.
+     */
+    override fun <T : Descriptor> register(descriptorClass: KClass<T>, provider: DescriptorProvider<*>) {
+        this.registered[descriptorClass] = provider
+    }
+
+    /**
+     * Obtains an [DescriptorProvider] for a particular [KClass] of [Descriptor], that has been registered with this [ConnectionProvider].
+     *
+     * @param descriptorClass The [KClass] of the [Descriptor] to lookup [DescriptorProvider] for.
+     * @return The registered [DescriptorProvider] .
+     */
+    @Suppress("UNCHECKED_CAST")
+    override fun <T : Descriptor> obtain(descriptorClass: KClass<T>): DescriptorProvider<T> {
+        val provider = this.registered[descriptorClass] ?: throw IllegalStateException("No DescriptorProvider registered for $descriptorClass.")
+        return provider as DescriptorProvider<T>
+    }
+}

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/Connection.kt

@@ -25,6 +25,13 @@ sealed interface Connection: Closeable {
     /** The name of the [Schema] managed by this [Connection]. */
     val schemaName: String
 
+    /**
+     * Executes the provided action within a transaction (if supported by the database).
+     *
+     * @param action The action to execute within the transaction.
+     */
+    fun <T> withTransaction(action: (Unit) -> T): T
+
     /**
      * Initializes the database layer with the [Schema] used by this [Connection].
      */