Merge pull request #89 from vitrivr/feature/harmoniseconfig

Harmonizes SchemaConfig to a named block structure. Thanks @v0idness for your contribution.
vitrivr · Aug 13, 2024 · c3895a8 · c3895a8
2 parents 4a082b3 + 311028d
commit c3895a8
Show file tree

Hide file tree

Showing 8 changed files with 50 additions and 52 deletions.
diff --git a/config-schema.json b/config-schema.json
@@ -1,32 +1,27 @@
 {
-  "schemas": [
-    {
-      "name": "vitrivr",
+  "schemas": {
+    "vitrivr": {
       "connection": {
         "database": "CottontailConnectionProvider",
         "parameters": {
           "host": "127.0.0.1",
           "port": "1865"
         }
       },
-      "fields": [
-        {
-          "name": "averagecolor",
+      "fields": {
+        "averagecolor": {
           "factory": "AverageColor"
         },
-        {
-          "name": "file",
+        "file": {
           "factory": "FileSourceMetadata"
         },
-        {
-          "name": "time",
+        "time": {
           "factory": "TemporalMetadata"
         },
-        {
-          "name": "video",
+        "video": {
           "factory": "VideoSourceMetadata"
         }
-      ],
+      },
       "resolvers": {
         "disk": {
           "factory": "DiskResolver",
@@ -35,18 +30,16 @@
           }
         }
       },
-      "exporters": [
-        {
-          "name": "thumbnail",
+      "exporters": {
+        "thumbnail": {
           "factory": "ThumbnailExporter",
           "resolverName": "disk",
           "parameters": {
             "maxSideResolution": "400",
             "mimeType": "JPG"
           }
         },
-        {
-          "name": "preview",
+        "preview": {
           "factory": "VideoPreviewExporter",
           "resolverName": "disk",
           "parameters": {
@@ -55,13 +48,12 @@
             "previewTimeSec": "5"
           }
         }
-      ],
-      "extractionPipelines": [
-        {
-          "name": "ingestion",
+      },
+      "extractionPipelines": {
+        "ingestion": {
           "path": "./config-ingestion.json"
         }
-      ]
+      }
     }
-  ]
+  }
 }
diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/schema/ExporterConfig.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/schema/ExporterConfig.kt
@@ -10,10 +10,6 @@ import org.vitrivr.engine.core.resolver.Resolver
  */
 @Serializable
 data class ExporterConfig(
-    /**
-     * The name of the [Exporter].
-     */
-    val name: String,
     /**
      * The simple or fully qualified class name of the [ExporterFactory]
      */

diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/schema/FieldConfig.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/schema/FieldConfig.kt
@@ -12,4 +12,4 @@ import org.vitrivr.engine.core.model.metamodel.Schema
  * @version 1.0.0
  */
 @Serializable
-data class FieldConfig(val name: String, val factory: String, val parameters: Map<String,String> = emptyMap(), val indexes: List<IndexConfig> = emptyList())
+data class FieldConfig(val factory: String, val parameters: Map<String,String> = emptyMap(), val indexes: List<IndexConfig> = emptyList())
diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/schema/PipelineConfig.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/schema/PipelineConfig.kt
@@ -4,6 +4,5 @@ import kotlinx.serialization.Serializable
 
 @Serializable
 data class PipelineConfig(
-    val name: String,
     val path: String,
 )
diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/schema/SchemaConfig.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/schema/SchemaConfig.kt
@@ -20,15 +20,15 @@ import java.nio.file.Paths
 @Serializable
 data class SchemaConfig(
     /** Name of the [Schema]. */
-    val name: String = "vitrivr",
+    var name: String = "vitrivr",
 
     /** The (database) [ConnectionConfig] for this [SchemaConfig]. */
     val connection: ConnectionConfig,
 
     /**
      * List of [FieldConfig]s that are part of this [SchemaConfig].
      */
-    val fields: List<FieldConfig>,
+    val fields: Map<String, FieldConfig>,
 
     /**
      * The list of [ResolverConfig]s that are part of this [SchemaConfig].
@@ -40,12 +40,12 @@ data class SchemaConfig(
      * List of [ExporterConfig]s that are part of this [SchemaConfig].
      * @see Exporter
      */
-    val exporters: List<ExporterConfig> = emptyList(),
+    val exporters: Map<String, ExporterConfig> = emptyMap(),
 
     /**
      * List of [PipelineConfig]s that are part of this [SchemaConfig].
      */
-    val extractionPipelines: List<PipelineConfig> = emptyList()
+    val extractionPipelines: Map<String, PipelineConfig> = emptyMap()
 ) {
 
     companion object {

diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/metamodel/SchemaManager.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/metamodel/SchemaManager.kt
@@ -49,38 +49,49 @@ class SchemaManager {
         /* Create new connection using reflection. */
         val connection = connectionProvider.openConnection(config.name, config.connection.parameters)
         val schema = Schema(config.name, connection)
-        config.fields.map {
-            val analyser = loadServiceForName<Analyser<*,*>>(it.factory) ?: throw IllegalArgumentException("Failed to find a factory implementation for '${it.factory}'.")
-            if(it.name.contains(".")){
+        config.fields.forEach { (fieldName, fieldConfig) ->
+            val analyser = loadServiceForName<Analyser<*,*>>(fieldConfig.factory) ?: throw IllegalArgumentException("Failed to find a factory implementation for '${fieldConfig.factory}'.")
+            if(fieldName.contains(".")){
                 throw IllegalArgumentException("Field names must not have a dot (.) in their name.")
             }
             @Suppress("UNCHECKED_CAST")
-            schema.addField(it.name, analyser as Analyser<ContentElement<*>, Descriptor>, it.parameters, it.indexes)
+            schema.addField(fieldName, analyser as Analyser<ContentElement<*>, Descriptor>, fieldConfig.parameters, fieldConfig.indexes)
         }
-        config.resolvers.map {
-            schema.addResolver(it.key, (loadServiceForName<ResolverFactory>(it.value.factory) ?: throw IllegalArgumentException("Failed to find resolver factory implementation for '${it.value.factory}'.")).newResolver(schema, it.value.parameters))
+        config.resolvers.forEach { (resolverName, resolverConfig) ->
+            schema.addResolver(resolverName, (loadServiceForName<ResolverFactory>(resolverConfig.factory) ?: throw IllegalArgumentException("Failed to find resolver factory implementation for '${resolverConfig.factory}'.")).newResolver(schema, resolverConfig.parameters))
         }
-        config.exporters.map {
+        config.exporters.forEach { (exporterName, exporterConfig) ->
             schema.addExporter(
-                it.name,
-                loadServiceForName<ExporterFactory>(it.factory) ?: throw IllegalArgumentException("Failed to find exporter factory implementation for '${it.factory}'."),
-                it.parameters,
-                it.resolverName
+                exporterName,
+                loadServiceForName<ExporterFactory>(exporterConfig.factory) ?: throw IllegalArgumentException("Failed to find exporter factory implementation for '${exporterConfig.factory}'."),
+                exporterConfig.parameters,
+                exporterConfig.resolverName
             )
         }
-        config.extractionPipelines.map {
-            val ingestionConfig = IngestionConfig.read(Paths.get(it.path))
-                ?: throw IllegalArgumentException("Failed to read pipeline configuration from '${it.path}'.")
+        config.extractionPipelines.forEach { (extractionPipelineName, extractionPipelineConfig) ->
+            val ingestionConfig = IngestionConfig.read(Paths.get(extractionPipelineConfig.path))
+                ?: throw IllegalArgumentException("Failed to read pipeline configuration from '${extractionPipelineConfig.path}'.")
             if (ingestionConfig.schema != schema.name) {
                 throw IllegalArgumentException("Schema name in pipeline configuration '${ingestionConfig.schema}' does not match schema name '${schema.name}'.")
             }
-            schema.addIngestionPipeline(it.name, ingestionConfig)
+            schema.addIngestionPipeline(extractionPipelineName, ingestionConfig)
         }
 
         /* Cache and return connection. */
         this.schemas[schema.name] = schema
     }
 
+    /**
+     * Sets the name for a [SchemaConfig], then calls load.
+     *
+     * @param name The name of the schema.
+     * @param config The [SchemaConfig] to which to assign the name and load.
+     */
+    fun load(name: String, config: SchemaConfig) {
+        config.name = name
+        load(config)
+    }
+
     /**
      * Lists all [Schema] managed by this [SchemaManager].
      *

diff --git a/vitrivr-engine-server/src/main/kotlin/org/vitrivr/engine/server/Main.kt b/vitrivr-engine-server/src/main/kotlin/org/vitrivr/engine/server/Main.kt
@@ -31,8 +31,8 @@ fun main(args: Array<String>) {
 
     /* Setup schema manager. */
     val manager = SchemaManager()
-    for (schema in config.schemas) {
-        manager.load(schema)
+    for ((name, schemaConfig) in config.schemas) {
+        manager.load(name, schemaConfig)
     }
 
     /* Execution server singleton for this instance. */

diff --git a/vitrivr-engine-server/src/main/kotlin/org/vitrivr/engine/server/config/ServerConfig.kt b/vitrivr-engine-server/src/main/kotlin/org/vitrivr/engine/server/config/ServerConfig.kt
@@ -19,7 +19,7 @@ data class ServerConfig(
     val api: ApiConfig = ApiConfig(),
 
     /** List of [SchemaConfig] managed by this [ServerConfig]. */
-    val schemas: List<SchemaConfig>
+    val schemas: Map<String, SchemaConfig>
 ) {
     companion object {
         /** Default path to fall back to. */