Adds JSONL connection

vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnection.kt

@@ -24,7 +24,7 @@ class JsonlConnection(
         return action(Unit)
     }
 
-    private val schemaRoot = File(root, schemaName)
+    internal val schemaRoot = File(root, schemaName)
 
     fun getFile(field: Schema.Field<*, *>) = File(schemaRoot, "${field.fieldName}.jsonl")
 
@@ -42,35 +42,13 @@ class JsonlConnection(
 
     override fun close() {
         writer.close()
-        reader.close()
     }
 
     companion object {
 
-        /** The name of the retrievable entity. */
-        const val RETRIEVABLE_ENTITY_NAME = "retrievable"
-
         /** The column name of a retrievable ID. */
         const val RETRIEVABLE_ID_COLUMN_NAME = "retrievableId"
 
-        /** The column name of a retrievable ID. */
-        const val RETRIEVABLE_TYPE_COLUMN_NAME = "type"
-
-        /** The name of the retrievable entity. */
-        const val RELATIONSHIP_ENTITY_NAME = "relationships"
-
-        /** The column name of a retrievable ID. */
-        const val SUBJECT_ID_COLUMN_NAME = "subjectId"
-
-        /** The column name of a retrievable ID. */
-        const val OBJECT_ID_COLUMN_NAME = "objectId"
-
-        /** The column name of a retrievable ID. */
-        const val PREDICATE_COLUMN_NAME = "predicate"
-
-        /** The prefix for descriptor entities. */
-        const val DESCRIPTOR_ENTITY_PREFIX = "descriptor"
-
         /** The column name of a descriptor ID. */
         const val DESCRIPTOR_ID_COLUMN_NAME = "descriptorId"
     }

vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/JsonlRelationship.kt

@@ -0,0 +1,13 @@
+package org.vitrivr.engine.database.jsonl.model
+
+import kotlinx.serialization.Serializable
+import org.vitrivr.engine.core.model.relationship.Relationship
+import org.vitrivr.engine.core.model.retrievable.RetrievableId
+
+@Serializable
+data class JsonlRelationship(val sub: RetrievableId, val pred: String, val obj: RetrievableId) {
+    constructor(relationship: Relationship) : this(relationship.subjectId, relationship.predicate, relationship.objectId)
+
+    fun toTriple() : Triple<RetrievableId, String, RetrievableId> = Triple(sub, pred, obj)
+
+}

vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/JsonlRetrievable.kt

@@ -0,0 +1,13 @@
+package org.vitrivr.engine.database.jsonl.model
+
+import kotlinx.serialization.Serializable
+import org.vitrivr.engine.core.model.retrievable.Retrievable
+import org.vitrivr.engine.core.model.retrievable.RetrievableId
+import org.vitrivr.engine.core.model.retrievable.Retrieved
+
+@Serializable
+data class JsonlRetrievable(val id: RetrievableId, val type: String) {
+    fun toRetrieved(): Retrieved = Retrieved(id, if (type.isEmpty()) null else type, false)
+
+    constructor(retrievable: Retrievable) : this(retrievable.id, retrievable.type ?: "")
+}

vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableInitializer.kt

@@ -2,22 +2,43 @@ package org.vitrivr.engine.database.jsonl.retrievable
 
 import org.vitrivr.engine.core.database.retrievable.RetrievableInitializer
 import org.vitrivr.engine.database.jsonl.JsonlConnection
+import org.vitrivr.engine.database.jsonl.LOGGER
+import java.io.File
+import java.io.IOException
 
 class JsonlRetrievableInitializer(private val connection: JsonlConnection) : RetrievableInitializer {
 
+    private val retrievableFile = File(connection.schemaRoot, "retrievables.jsonl")
+    private val connectionFile = File(connection.schemaRoot, "retrievable_connections.jsonl")
+
     override fun initialize() {
-        TODO("Not yet implemented")
+        try {
+            connection.schemaRoot.mkdirs()
+            retrievableFile.createNewFile()
+            connectionFile.createNewFile()
+        } catch (ioe: IOException) {
+            LOGGER.error(ioe) { "Cannot initialize '${connection.schemaRoot.absolutePath}'" }
+        } catch (se: SecurityException) {
+            LOGGER.error(se) { "Cannot initialize '${connection.schemaRoot.absolutePath}'" }
+        }
     }
 
     override fun deinitialize() {
-        TODO("Not yet implemented")
+        /* nop */
     }
 
     override fun isInitialized(): Boolean {
         TODO("Not yet implemented")
     }
 
     override fun truncate() {
-        TODO("Not yet implemented")
+        try {
+            retrievableFile.delete()
+            connectionFile.delete()
+        } catch (ioe: IOException) {
+            LOGGER.error(ioe) { "Cannot delete '${connection.schemaRoot.absolutePath}'" }
+        } catch (se: SecurityException) {
+            LOGGER.error(se) { "Cannot delete '${connection.schemaRoot.absolutePath}'" }
+        }
     }
 }

vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableReader.kt

@@ -1,41 +1,72 @@
 package org.vitrivr.engine.database.jsonl.retrievable
 
+import kotlinx.serialization.SerializationException
+import kotlinx.serialization.json.Json
 import org.vitrivr.engine.core.database.retrievable.RetrievableReader
 import org.vitrivr.engine.core.model.retrievable.Retrievable
 import org.vitrivr.engine.core.model.retrievable.RetrievableId
 import org.vitrivr.engine.database.jsonl.JsonlConnection
+import org.vitrivr.engine.database.jsonl.LOGGER
+import org.vitrivr.engine.database.jsonl.model.JsonlRelationship
+import org.vitrivr.engine.database.jsonl.model.JsonlRetrievable
+import java.io.BufferedReader
+import java.io.File
+import java.io.FileReader
 
 class JsonlRetrievableReader(override val connection: JsonlConnection) : RetrievableReader {
 
-    override fun get(id: RetrievableId): Retrievable? {
-        TODO("Not yet implemented")
-    }
+    private val retrievableFile = File(connection.schemaRoot, "retrievables.jsonl")
+    private val connectionFile = File(connection.schemaRoot, "retrievable_connections.jsonl")
 
-    override fun exists(id: RetrievableId): Boolean {
-        TODO("Not yet implemented")
-    }
+    override fun get(id: RetrievableId): Retrievable? = getAll().firstOrNull { it.id == id }
+
+    override fun exists(id: RetrievableId): Boolean = get(id) != null
 
     override fun getAll(ids: Iterable<RetrievableId>): Sequence<Retrievable> {
-        TODO("Not yet implemented")
+        val idSet = ids.toSet()
+        return getAll().filter { idSet.contains(it.id) }
     }
 
     override fun getConnections(
         subjectIds: Collection<RetrievableId>,
         predicates: Collection<String>,
         objectIds: Collection<RetrievableId>
     ): Sequence<Triple<RetrievableId, String, RetrievableId>> {
-        TODO("Not yet implemented")
-    }
+        val subIds = subjectIds.toSet()
+        val predIds = predicates.toSet()
+        val objIds = objectIds.toSet()
 
-    override fun getAll(): Sequence<Retrievable> {
-        TODO("Not yet implemented")
+        return BufferedReader(FileReader(connectionFile)).lineSequence().mapNotNull {
+            try {
+                Json.decodeFromString<JsonlRelationship>(it)
+            } catch (se: SerializationException) {
+                LOGGER.error(se) { "Error during deserialization" }
+                null
+            } catch (ie: IllegalArgumentException) {
+                LOGGER.error(ie) { "Error during deserialization" }
+                null
+            }
+        }.filter {
+            (subIds.isEmpty() || subIds.contains(it.sub)) &&
+                    (predIds.isEmpty() || predIds.contains(it.pred)) &&
+                    (objIds.isEmpty() || objIds.contains(it.obj))
+        }.map { it.toTriple() }
     }
 
-    override fun count(): Long {
-        TODO("Not yet implemented")
+    override fun getAll(): Sequence<Retrievable> {
+        return BufferedReader(FileReader(retrievableFile)).lineSequence().mapNotNull {
+            try {
+                Json.decodeFromString<JsonlRetrievable>(it).toRetrieved()
+            } catch (se: SerializationException) {
+                LOGGER.error(se) { "Error during deserialization" }
+                null
+            } catch (ie: IllegalArgumentException) {
+                LOGGER.error(ie) { "Error during deserialization" }
+                null
+            }
+        }
     }
 
-    fun close() {
+    override fun count(): Long = getAll().count().toLong()
 
-    }
 }

vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableWriter.kt

@@ -1,15 +1,28 @@
 package org.vitrivr.engine.database.jsonl.retrievable
 
+import kotlinx.serialization.encodeToString
+import kotlinx.serialization.json.Json
 import org.vitrivr.engine.core.database.retrievable.RetrievableWriter
 import org.vitrivr.engine.core.model.relationship.Relationship
 import org.vitrivr.engine.core.model.retrievable.Retrievable
 import org.vitrivr.engine.database.jsonl.JsonlConnection
 import org.vitrivr.engine.database.jsonl.LOGGER
+import org.vitrivr.engine.database.jsonl.model.JsonlRelationship
+import org.vitrivr.engine.database.jsonl.model.JsonlRetrievable
+import java.io.File
+import java.io.FileWriter
 
-class JsonlRetrievableWriter(override val connection: JsonlConnection) : RetrievableWriter {
+class JsonlRetrievableWriter(override val connection: JsonlConnection) : RetrievableWriter, AutoCloseable {
 
+    private val retrievableWriter = FileWriter(File(connection.schemaRoot, "retrievables.jsonl"), true)
+    private val connectionWriter = FileWriter(File(connection.schemaRoot, "retrievable_connections.jsonl"), true)
+
+    @Synchronized
     override fun connect(relationship: Relationship): Boolean {
-        TODO("Not yet implemented")
+        connectionWriter.write(Json.encodeToString(JsonlRelationship(relationship)))
+        connectionWriter.write("\n")
+        connectionWriter.flush()
+        return true
     }
 
     override fun connectAll(relationships: Iterable<Relationship>): Boolean {
@@ -27,8 +40,12 @@ class JsonlRetrievableWriter(override val connection: JsonlConnection) : Retriev
         return false
     }
 
+    @Synchronized
     override fun add(item: Retrievable): Boolean {
-        TODO("Not yet implemented")
+        retrievableWriter.write(Json.encodeToString(JsonlRetrievable(item)))
+        retrievableWriter.write("\n")
+        retrievableWriter.flush()
+        return true
     }
 
     override fun addAll(items: Iterable<Retrievable>): Boolean {
@@ -51,7 +68,8 @@ class JsonlRetrievableWriter(override val connection: JsonlConnection) : Retriev
         return false
     }
 
-    fun close() {
-
+    override fun close() {
+        retrievableWriter.close()
+        connectionWriter.close()
     }
 }

vitrivr-engine-index/src/main/resources/META-INF/services/org.vitrivr.engine.core.database.ConnectionProvider

@@ -0,0 +1 @@
+org.vitrivr.engine.database.jsonl.JsonlConnectionProvider

vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/Utilities.kt

@@ -50,6 +50,7 @@ internal fun Type.toSql(): Int = when (this) {
     Type.Datetime -> JDBCType.DATE
     Type.String -> JDBCType.VARCHAR
     Type.Text -> JDBCType.CLOB
+    Type.UUID -> JDBCType.OTHER
     is Type.BooleanVector -> JDBCType.ARRAY
     is Type.DoubleVector -> JDBCType.ARRAY
     is Type.FloatVector -> JDBCType.ARRAY

vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/PgDescriptorInitializer.kt

@@ -54,6 +54,7 @@ open class PgDescriptorInitializer<D : Descriptor>(final override val field: Sch
                 Type.Float -> statement.append("\"${field.name}\" real, ")
                 Type.Double -> statement.append("\"${field.name}\" double precision, ")
                 Type.Datetime -> statement.append("\"${field.name}\" datetime, ")
+                Type.UUID -> statement.append("\"${field.name}\" uuid, ")
                 is Type.BooleanVector -> statement.append("\"${field.name}\" bit(${field.type.dimensions}), ")
                 is Type.DoubleVector -> statement.append("\"${field.name}\" vector(${field.type.dimensions}), ")
                 is Type.FloatVector -> statement.append("\"${field.name}\" vector(${field.type.dimensions}), ")

vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/struct/StructDescriptorReader.kt

@@ -65,6 +65,7 @@ class StructDescriptorReader(field: Schema.Field<*, StructDescriptor>, connectio
                 Type.Float -> result.getFloat(field.name).let { Value.Float(it) }
                 Type.Double -> result.getDouble(field.name).let { Value.Double(it) }
                 Type.Datetime -> result.getDate(field.name).toInstant().let { Value.DateTime(Date(it.toEpochMilli())) }
+                Type.UUID -> result.getObject(field.name, UUID::class.java).let { Value.UUIDValue(it) }
                 is Type.BooleanVector -> result.getObject(field.name, PgBitVector::class.java).toBooleanVector()
                 is Type.IntVector -> result.getObject(field.name, PgVector::class.java)?.toIntVector()
                 is Type.LongVector -> result.getObject(field.name, PgVector::class.java)?.toLongVector()