From f1c31a7b8bf1eeddf7416e4e941f79a58419a2cc Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Sat, 27 Jul 2024 16:37:05 +0200 Subject: [PATCH 01/11] Extended Type system and serialization mechanism to be used in JSON serialization of descriptors and retrievables --- .../source/exif/ExifMetadataExtractor.kt | 1 + .../engine/core/model/descriptor/Attribute.kt | 2 + .../core/model/descriptor/Descriptor.kt | 4 +- .../descriptor/scalar/FloatDescriptor.kt | 2 +- .../model/descriptor/scalar/LongDescriptor.kt | 2 +- .../descriptor/scalar/ScalarDescriptor.kt | 2 + .../core/model/retrievable/Retrievable.kt | 4 +- .../core/model/serializer/DateSerializer.kt | 14 +++ .../model/serializer/DateTimeSerializer.kt | 15 +++ .../core/model/serializer/UUIDSerializer.kt | 14 +++ .../vitrivr/engine/core/model/types/Type.kt | 25 ++++ .../vitrivr/engine/core/model/types/Value.kt | 94 ++++++++++++-- vitrivr-engine-index/build.gradle | 1 + .../database/jsonl/AttributeContainer.kt | 13 ++ .../database/jsonl/AttributeContainerList.kt | 6 + .../engine/database/jsonl/JsonlConnection.kt | 75 +++++++++++ .../database/jsonl/JsonlConnectionProvider.kt | 56 +++++++++ .../database/jsonl/JsonlDescriptorProvider.kt | 20 +++ .../engine/database/jsonl/JsonlInitializer.kt | 39 ++++++ .../engine/database/jsonl/JsonlProvider.kt | 24 ++++ .../engine/database/jsonl/JsonlReader.kt | 48 +++++++ .../jsonl/JsonlRetrievableInitializer.kt | 22 ++++ .../database/jsonl/JsonlRetrievableReader.kt | 41 ++++++ .../database/jsonl/JsonlRetrievableWriter.kt | 55 ++++++++ .../engine/database/jsonl/JsonlWriter.kt | 71 +++++++++++ .../engine/database/jsonl/ValueContainer.kt | 117 ++++++++++++++++++ .../engine/plugin/cottontaildb/Common.kt | 2 + 27 files changed, 755 insertions(+), 14 deletions(-) create mode 100644 vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/serializer/DateSerializer.kt create mode 100644 vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/serializer/DateTimeSerializer.kt create mode 100644 vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/serializer/UUIDSerializer.kt create mode 100644 vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/AttributeContainer.kt create mode 100644 vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/AttributeContainerList.kt create mode 100644 vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnection.kt create mode 100644 vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnectionProvider.kt create mode 100644 vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlDescriptorProvider.kt create mode 100644 vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlInitializer.kt create mode 100644 vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlProvider.kt create mode 100644 vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlReader.kt create mode 100644 vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlRetrievableInitializer.kt create mode 100644 vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlRetrievableReader.kt create mode 100644 vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlRetrievableWriter.kt create mode 100644 vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlWriter.kt create mode 100644 vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/ValueContainer.kt diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/metadata/source/exif/ExifMetadataExtractor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/metadata/source/exif/ExifMetadataExtractor.kt index c24eee29..4c9fab6e 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/metadata/source/exif/ExifMetadataExtractor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/metadata/source/exif/ExifMetadataExtractor.kt @@ -58,6 +58,7 @@ private fun convertType(directory: Directory, tagType: Int, type: Type): Value<* Type.Short -> Value.Short(directory.getObject(tagType) as Short) Type.String -> Value.String(directory.getString(tagType)) Type.Text -> Value.String(directory.getString(tagType)) + Type.UUID -> Value.UUIDValue(UUID.fromString(directory.getString(tagType))) is Type.BooleanVector -> throw IllegalArgumentException("Unsupported type: $type") is Type.DoubleVector -> throw IllegalArgumentException("Unsupported type: $type") is Type.FloatVector -> throw IllegalArgumentException("Unsupported type: $type") diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/Attribute.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/Attribute.kt index df1a6930..051a0b43 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/Attribute.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/Attribute.kt @@ -1,5 +1,6 @@ package org.vitrivr.engine.core.model.descriptor +import kotlinx.serialization.Serializable import org.vitrivr.engine.core.model.types.Type /** The name of an attribute. */ @@ -12,4 +13,5 @@ typealias AttributeName = String * @author Ralph Gasser * @version 1.0.0 */ +@Serializable data class Attribute(val name: AttributeName, val type: Type, val nullable: Boolean = false) \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/Descriptor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/Descriptor.kt index bfbe2939..3a64f459 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/Descriptor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/Descriptor.kt @@ -1,14 +1,16 @@ package org.vitrivr.engine.core.model.descriptor +import kotlinx.serialization.Serializable import org.vitrivr.engine.core.model.Persistable import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.retrievable.Retrievable import org.vitrivr.engine.core.model.retrievable.RetrievableId +import org.vitrivr.engine.core.model.serializer.UUIDSerializer import org.vitrivr.engine.core.model.types.Value import java.util.* /** A typealias to identify the [UUID] identifying a [Descriptor]. */ -typealias DescriptorId = UUID +typealias DescriptorId = @Serializable(UUIDSerializer::class) UUID /** * A [Persistable] [Descriptor] that can be used to describe a [Retrievable]. diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/FloatDescriptor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/FloatDescriptor.kt index b87fa80c..00efb021 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/FloatDescriptor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/FloatDescriptor.kt @@ -1,5 +1,6 @@ package org.vitrivr.engine.core.model.descriptor.scalar +import kotlinx.serialization.Serializable import org.vitrivr.engine.core.model.descriptor.Attribute import org.vitrivr.engine.core.model.descriptor.DescriptorId import org.vitrivr.engine.core.model.descriptor.scalar.ScalarDescriptor.Companion.VALUE_ATTRIBUTE_NAME @@ -14,7 +15,6 @@ import org.vitrivr.engine.core.model.types.Value * @author Ralph Gasser * @version 1.0.0 */ - data class FloatDescriptor( override var id: DescriptorId, override var retrievableId: RetrievableId?, diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/LongDescriptor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/LongDescriptor.kt index a472150d..6e6fb06d 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/LongDescriptor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/LongDescriptor.kt @@ -1,5 +1,6 @@ package org.vitrivr.engine.core.model.descriptor.scalar + import org.vitrivr.engine.core.model.descriptor.Attribute import org.vitrivr.engine.core.model.descriptor.DescriptorId import org.vitrivr.engine.core.model.descriptor.scalar.ScalarDescriptor.Companion.VALUE_ATTRIBUTE_NAME @@ -14,7 +15,6 @@ import org.vitrivr.engine.core.model.types.Value * @author Ralph Gasser * @version 1.0.0 */ - data class LongDescriptor( override var id: DescriptorId, override var retrievableId: RetrievableId?, diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/ScalarDescriptor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/ScalarDescriptor.kt index 452a369f..508dd38b 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/ScalarDescriptor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/ScalarDescriptor.kt @@ -1,5 +1,6 @@ package org.vitrivr.engine.core.model.descriptor.scalar +import kotlinx.serialization.Serializable import org.vitrivr.engine.core.model.descriptor.AttributeName import org.vitrivr.engine.core.model.descriptor.Descriptor import org.vitrivr.engine.core.model.types.Value @@ -10,6 +11,7 @@ import org.vitrivr.engine.core.model.types.Value * @author Ralph Gasser * @version 1.1.0 */ +@Serializable sealed interface ScalarDescriptor> : Descriptor { companion object { diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/retrievable/Retrievable.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/retrievable/Retrievable.kt index c04a2577..86cdc765 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/retrievable/Retrievable.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/retrievable/Retrievable.kt @@ -1,15 +1,17 @@ package org.vitrivr.engine.core.model.retrievable +import kotlinx.serialization.Serializable import org.vitrivr.engine.core.model.Persistable import org.vitrivr.engine.core.model.content.element.ContentElement import org.vitrivr.engine.core.model.descriptor.Descriptor import org.vitrivr.engine.core.model.relationship.Relationship import org.vitrivr.engine.core.model.retrievable.attributes.RetrievableAttribute +import org.vitrivr.engine.core.model.serializer.UUIDSerializer import java.util.* import java.util.function.Predicate /** A typealias to identify the [UUID] identifying a [Retrievable]. */ -typealias RetrievableId = UUID +typealias RetrievableId = @Serializable(UUIDSerializer::class) UUID /** * A [Persistable] and [Retrievable] unit of information stored by vitrivr. diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/serializer/DateSerializer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/serializer/DateSerializer.kt new file mode 100644 index 00000000..7d78a52a --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/serializer/DateSerializer.kt @@ -0,0 +1,14 @@ +package org.vitrivr.engine.core.model.serializer + +import kotlinx.serialization.KSerializer +import kotlinx.serialization.descriptors.PrimitiveKind +import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor +import kotlinx.serialization.encoding.Decoder +import kotlinx.serialization.encoding.Encoder +import java.util.* + +object DateSerializer: KSerializer { + override val descriptor = PrimitiveSerialDescriptor("Date", PrimitiveKind.LONG) + override fun serialize(encoder: Encoder, value: Date) = encoder.encodeLong(value.time) + override fun deserialize(decoder: Decoder): Date = Date(decoder.decodeLong()) +} \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/serializer/DateTimeSerializer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/serializer/DateTimeSerializer.kt new file mode 100644 index 00000000..3a923c02 --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/serializer/DateTimeSerializer.kt @@ -0,0 +1,15 @@ +package org.vitrivr.engine.core.model.serializer + +import kotlinx.serialization.KSerializer +import kotlinx.serialization.descriptors.PrimitiveKind +import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor +import kotlinx.serialization.encoding.Decoder +import kotlinx.serialization.encoding.Encoder +import org.vitrivr.engine.core.model.types.Value +import java.util.* + +object DateTimeSerializer: KSerializer { + override val descriptor = PrimitiveSerialDescriptor("DateTime", PrimitiveKind.LONG) + override fun serialize(encoder: Encoder, value: Value.DateTime) = encoder.encodeLong(value.value.time) + override fun deserialize(decoder: Decoder): Value.DateTime = Value.DateTime(Date(decoder.decodeLong())) +} \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/serializer/UUIDSerializer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/serializer/UUIDSerializer.kt new file mode 100644 index 00000000..8cffbf8c --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/serializer/UUIDSerializer.kt @@ -0,0 +1,14 @@ +package org.vitrivr.engine.core.model.serializer + +import kotlinx.serialization.KSerializer +import kotlinx.serialization.descriptors.PrimitiveKind +import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor +import kotlinx.serialization.encoding.Decoder +import kotlinx.serialization.encoding.Encoder +import java.util.UUID + +object UUIDSerializer: KSerializer { + override val descriptor = PrimitiveSerialDescriptor("UUID", PrimitiveKind.STRING) + override fun serialize(encoder: Encoder, value: UUID) = encoder.encodeString(value.toString()) + override fun deserialize(decoder: Decoder): UUID = UUID.fromString(decoder.decodeString()) +} \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/types/Type.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/types/Type.kt index 8f3928aa..ff4d7b39 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/types/Type.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/types/Type.kt @@ -1,5 +1,6 @@ package org.vitrivr.engine.core.model.types +import kotlinx.serialization.Serializable import java.util.* /** @@ -55,6 +56,7 @@ sealed interface Type { * * Primarily needed, since simple strings and longer texts are treated differently by certain databases. */ + @Serializable data object Text : Type { override val dimensions: kotlin.Int = 1 override fun defaultValue(): Value<*> = Value.String("") @@ -65,6 +67,7 @@ sealed interface Type { * * Primarily needed, since simple strings and longer texts are treated differently by certain databases. */ + @Serializable data object String : Type { override val dimensions: kotlin.Int = 1 override fun defaultValue(): Value<*> = Value.String("") @@ -73,6 +76,7 @@ sealed interface Type { /** * A [Type] that represents a [Boolean] value. */ + @Serializable data object Boolean : Type { override val dimensions: kotlin.Int = 1 override fun defaultValue(): Value<*> = Value.Boolean(false) @@ -81,6 +85,7 @@ sealed interface Type { /** * A [Type] that represents a [Byte] value. */ + @Serializable data object Byte : Type { override val dimensions: kotlin.Int = 1 override fun defaultValue(): Value<*> = Value.Byte(0) @@ -89,6 +94,7 @@ sealed interface Type { /** * A [Type] that represents a [Short] value. */ + @Serializable data object Short : Type { override val dimensions: kotlin.Int = 1 override fun defaultValue(): Value<*> = Value.Short(0) @@ -97,6 +103,7 @@ sealed interface Type { /** * A [Type] that represents a [Int] value. */ + @Serializable data object Int : Type { override val dimensions: kotlin.Int = 1 override fun defaultValue(): Value<*> = Value.Int(0) @@ -105,6 +112,7 @@ sealed interface Type { /** * A [Type] that represents a [Long] value. */ + @Serializable data object Long : Type { override val dimensions: kotlin.Int = 1 override fun defaultValue(): Value<*> = Value.Long(0L) @@ -113,6 +121,7 @@ sealed interface Type { /** * A [Type] that represents a [Float] value. */ + @Serializable data object Float : Type { override val dimensions: kotlin.Int = 1 override fun defaultValue(): Value<*> = Value.Float(0.0f) @@ -121,6 +130,7 @@ sealed interface Type { /** * A [Type] that represents a [Double] value. */ + @Serializable data object Double : Type { override val dimensions: kotlin.Int = 1 override fun defaultValue(): Value<*> = Value.Double(0.0) @@ -129,14 +139,25 @@ sealed interface Type { /** * A [Type] that represents a [Datetime] value. */ + @Serializable data object Datetime : Type { override val dimensions: kotlin.Int = 1 override fun defaultValue(): Value<*> = Value.DateTime(Date()) } + /** + * A [Type] that represents a [UUID] value. + */ + @Serializable + data object UUID : Type { + override val dimensions: kotlin.Int = 1 + override fun defaultValue(): Value<*> = Value.UUIDValue(java.util.UUID(0L, 0L)) + } + /** * A [Type] that represents a [BooleanVector] value. */ + @Serializable data class BooleanVector(override val dimensions: kotlin.Int) : Type { override fun defaultValue(): Value<*> = Value.BooleanVector(this.dimensions) } @@ -144,6 +165,7 @@ sealed interface Type { /** * A [Type] that represents a [IntVector] value. */ + @Serializable data class IntVector(override val dimensions: kotlin.Int) : Type { override fun defaultValue(): Value<*> = Value.IntVector(this.dimensions) } @@ -151,6 +173,7 @@ sealed interface Type { /** * A [Type] that represents a [LongVector] value. */ + @Serializable data class LongVector(override val dimensions: kotlin.Int) : Type { override fun defaultValue(): Value<*> = Value.LongVector(this.dimensions) } @@ -158,6 +181,7 @@ sealed interface Type { /** * A [Type] that represents a [FloatVector] value. */ + @Serializable data class FloatVector(override val dimensions: kotlin.Int) : Type { override fun defaultValue(): Value<*> = Value.FloatVector(this.dimensions) } @@ -165,6 +189,7 @@ sealed interface Type { /** * A [Type] that represents a [DoubleVector] value. */ + @Serializable data class DoubleVector(override val dimensions: kotlin.Int) : Type { override fun defaultValue(): Value<*> = Value.DoubleVector(this.dimensions) } diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/types/Value.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/types/Value.kt index 5bd1dbbb..65d85950 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/types/Value.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/types/Value.kt @@ -1,5 +1,7 @@ package org.vitrivr.engine.core.model.types +import kotlinx.serialization.Serializable +import org.vitrivr.engine.core.model.serializer.DateTimeSerializer import java.util.* /** @@ -10,6 +12,7 @@ import java.util.* * @author Ralph Gasser * @version 1.0.0 */ +@Serializable() sealed interface Value { companion object { @@ -34,6 +37,7 @@ sealed interface Value { is LongArray -> LongVector(value) is IntArray -> IntVector(value) is Date -> DateTime(value) + is UUID -> UUIDValue(value) else -> throw IllegalArgumentException("Unsupported data type.") } } @@ -41,38 +45,87 @@ sealed interface Value { /** The actual, primitive value. */ val value: T + /** Reference to the [Type] of the Value*/ + val type: Type + sealed interface ScalarValue: Value @JvmInline - value class String(override val value: kotlin.String) : ScalarValue + @Serializable + value class String(override val value: kotlin.String) : ScalarValue { + override val type: Type + get() = Type.String + } @JvmInline - value class Text(override val value: kotlin.String) : ScalarValue + @Serializable + value class Text(override val value: kotlin.String) : ScalarValue { + override val type: Type + get() = Type.Text + } @JvmInline - value class Boolean(override val value: kotlin.Boolean) : ScalarValue + @Serializable + value class Boolean(override val value: kotlin.Boolean) : ScalarValue { + override val type: Type + get() = Type.Boolean + } @JvmInline - value class Byte(override val value: kotlin.Byte) : ScalarValue + @Serializable + value class Byte(override val value: kotlin.Byte) : ScalarValue { + override val type: Type + get() = Type.Byte + } @JvmInline - value class Short(override val value: kotlin.Short) : ScalarValue + @Serializable + value class Short(override val value: kotlin.Short) : ScalarValue { + override val type: Type + get() = Type.Short + } @JvmInline - value class Int(override val value: kotlin.Int) : ScalarValue + @Serializable + value class Int(override val value: kotlin.Int) : ScalarValue { + override val type: Type + get() = Type.Int + } @JvmInline - value class Long(override val value: kotlin.Long) : ScalarValue + @Serializable + value class Long(override val value: kotlin.Long) : ScalarValue { + override val type: Type + get() = Type.Long + } @JvmInline - value class Float(override val value: kotlin.Float) : ScalarValue + @Serializable + value class Float(override val value: kotlin.Float) : ScalarValue { + override val type: Type + get() = Type.Float + } @JvmInline - value class Double(override val value: kotlin.Double) : ScalarValue + @Serializable + value class Double(override val value: kotlin.Double) : ScalarValue { + override val type: Type + get() = Type.Double + } @JvmInline - value class DateTime(override val value: Date) : ScalarValue + @Serializable(with = DateTimeSerializer::class) + value class DateTime(override val value: Date) : ScalarValue { + override val type: Type + get() = Type.Datetime + } + + @JvmInline + value class UUIDValue(override val value: UUID) : ScalarValue { + override val type: Type + get() = Type.UUID + } /** * A [Vector] in vitrivr-engine maps primitive data types. @@ -84,50 +137,71 @@ sealed interface Value { * @property value The actual, primitive value. * @constructor Creates a new [Vector] with the given [value]. */ + @Serializable sealed interface Vector : Value { val size: kotlin.Int } @JvmInline + @Serializable value class BooleanVector(override val value: BooleanArray) : Vector { constructor(size: kotlin.Int, init: (kotlin.Int) -> kotlin.Boolean = { false }) : this(BooleanArray(size, init)) override val size: kotlin.Int get() = this.value.size + + override val type: Type + get() = Type.BooleanVector(size) } @JvmInline + @Serializable value class IntVector(override val value: IntArray) : Vector { constructor(size: kotlin.Int, init: (kotlin.Int) -> kotlin.Int = { 0 }) : this(IntArray(size, init)) override val size: kotlin.Int get() = this.value.size + + override val type: Type + get() = Type.IntVector(size) } @JvmInline + @Serializable value class LongVector(override val value: LongArray) : Vector { constructor(size: kotlin.Int, init: (kotlin.Int) -> kotlin.Long = { 0L }) : this(LongArray(size, init)) override val size: kotlin.Int get() = this.value.size + + override val type: Type + get() = Type.LongVector(size) } @JvmInline + @Serializable value class FloatVector(override val value: FloatArray) : Vector { constructor(size: kotlin.Int, init: (kotlin.Int) -> kotlin.Float = { 0.0f }) : this(FloatArray(size, init)) override val size: kotlin.Int get() = this.value.size + + override val type: Type + get() = Type.FloatVector(size) } @JvmInline + @Serializable value class DoubleVector(override val value: DoubleArray) : Vector { constructor(size: kotlin.Int, init: (kotlin.Int) -> kotlin.Double = { 0.0 }) : this(DoubleArray(size, init)) override val size: kotlin.Int get() = this.value.size + + override val type: Type + get() = Type.DoubleVector(size) } } diff --git a/vitrivr-engine-index/build.gradle b/vitrivr-engine-index/build.gradle index afefe7ef..9efc97a4 100644 --- a/vitrivr-engine-index/build.gradle +++ b/vitrivr-engine-index/build.gradle @@ -13,6 +13,7 @@ dependencies { /** ScrImage (used for image resizing). */ implementation group: 'com.sksamuel.scrimage', name: 'scrimage-core', version: version_scrimage + } /* Publication of vitrivr engine index to Maven Central. */ diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/AttributeContainer.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/AttributeContainer.kt new file mode 100644 index 00000000..727d0e96 --- /dev/null +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/AttributeContainer.kt @@ -0,0 +1,13 @@ +package org.vitrivr.engine.database.jsonl + +import kotlinx.serialization.Serializable +import org.vitrivr.engine.core.model.descriptor.Attribute +import org.vitrivr.engine.core.model.types.Value + +@Serializable +data class AttributeContainer(val attribute: Attribute, val value: ValueContainer?) { + constructor(attribute: Attribute, value: Value<*>?) : this( + attribute, + if (value == null) null else ValueContainer.fromValue(value) + ) +} diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/AttributeContainerList.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/AttributeContainerList.kt new file mode 100644 index 00000000..2d1b12a4 --- /dev/null +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/AttributeContainerList.kt @@ -0,0 +1,6 @@ +package org.vitrivr.engine.database.jsonl + +import kotlinx.serialization.Serializable + +@Serializable +data class AttributeContainerList(val list: List) diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnection.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnection.kt new file mode 100644 index 00000000..2dfbc1c3 --- /dev/null +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnection.kt @@ -0,0 +1,75 @@ +package org.vitrivr.engine.database.jsonl + +import io.github.oshai.kotlinlogging.KotlinLogging.logger +import org.vitrivr.engine.core.database.AbstractConnection +import org.vitrivr.engine.core.database.retrievable.RetrievableInitializer +import org.vitrivr.engine.core.database.retrievable.RetrievableReader +import org.vitrivr.engine.core.database.retrievable.RetrievableWriter +import org.vitrivr.engine.core.model.metamodel.Schema +import java.io.File + +internal val LOGGER = logger("org.vitrivr.engine.database.jsonl.JsonlConnection") + +class JsonlConnection( + override val schemaName: String, + connectionProvider: JsonlConnectionProvider, + private val root: File +) : AbstractConnection(schemaName, connectionProvider) { + + override fun withTransaction(action: (Unit) -> T): T { + LOGGER.warn { "Transactions are not supported by the JsonlConnection" } + return action(Unit) + } + + private val schemaRoot = File(root, schemaName) + + fun getFile(field: Schema.Field<*, *>) = File(schemaRoot, "${field.fieldName}.jsonl") + + override fun getRetrievableInitializer(): RetrievableInitializer = JsonlRetrievableInitializer(this) + + private val writer: JsonlRetrievableWriter by lazy { JsonlRetrievableWriter(this) } + + override fun getRetrievableWriter(): RetrievableWriter = writer + + private val reader: JsonlRetrievableReader by lazy { JsonlRetrievableReader(this) } + + override fun getRetrievableReader(): RetrievableReader = reader + + override fun description(): String = "JsonlConnection on '${root.absolutePath}'" + + override fun close() { + writer.close() + reader.close() + } + + companion object { + + /** The name of the retrievable entity. */ + const val RETRIEVABLE_ENTITY_NAME = "retrievable" + + /** The column name of a retrievable ID. */ + const val RETRIEVABLE_ID_COLUMN_NAME = "retrievableId" + + /** The column name of a retrievable ID. */ + const val RETRIEVABLE_TYPE_COLUMN_NAME = "type" + + /** The name of the retrievable entity. */ + const val RELATIONSHIP_ENTITY_NAME = "relationships" + + /** The column name of a retrievable ID. */ + const val SUBJECT_ID_COLUMN_NAME = "subjectId" + + /** The column name of a retrievable ID. */ + const val OBJECT_ID_COLUMN_NAME = "objectId" + + /** The column name of a retrievable ID. */ + const val PREDICATE_COLUMN_NAME = "predicate" + + /** The prefix for descriptor entities. */ + const val DESCRIPTOR_ENTITY_PREFIX = "descriptor" + + /** The column name of a descriptor ID. */ + const val DESCRIPTOR_ID_COLUMN_NAME = "descriptorId" + } + +} \ No newline at end of file diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnectionProvider.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnectionProvider.kt new file mode 100644 index 00000000..207332cf --- /dev/null +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnectionProvider.kt @@ -0,0 +1,56 @@ +package org.vitrivr.engine.database.jsonl + +import org.vitrivr.engine.core.database.AbstractConnectionProvider +import org.vitrivr.engine.core.database.Connection +import org.vitrivr.engine.core.model.descriptor.scalar.* +import org.vitrivr.engine.core.model.descriptor.struct.LabelDescriptor +import org.vitrivr.engine.core.model.descriptor.struct.MapStructDescriptor +import org.vitrivr.engine.core.model.descriptor.struct.metadata.MediaDimensionsDescriptor +import org.vitrivr.engine.core.model.descriptor.struct.metadata.Rectangle2DMetadataDescriptor +import org.vitrivr.engine.core.model.descriptor.struct.metadata.TemporalMetadataDescriptor +import org.vitrivr.engine.core.model.descriptor.struct.metadata.source.FileSourceMetadataDescriptor +import org.vitrivr.engine.core.model.descriptor.struct.metadata.source.VideoSourceMetadataDescriptor +import org.vitrivr.engine.core.model.descriptor.vector.* +import java.io.File + + +class JsonlConnectionProvider() : AbstractConnectionProvider() { + + override val databaseName = "JSONL File Connection" + override val version = "1.0.0" + + override fun openConnection(schemaName: String, parameters: Map): Connection { + + val rootPath = parameters["root"] ?: "." + + return JsonlConnection(schemaName, this, File(rootPath)) + + } + + override fun initialize() { + /* Scalar descriptors. */ + this.register(BooleanDescriptor::class, JsonlProvider()) + this.register(IntDescriptor::class, JsonlProvider()) + this.register(LongDescriptor::class, JsonlProvider()) + this.register(FloatDescriptor::class, JsonlProvider()) + this.register(DoubleDescriptor::class, JsonlProvider()) + this.register(StringDescriptor::class, JsonlProvider()) + + /* Vector descriptors. */ + this.register(BooleanVectorDescriptor::class, JsonlProvider()) + this.register(IntVectorDescriptor::class, JsonlProvider()) + this.register(LongVectorDescriptor::class, JsonlProvider()) + this.register(FloatVectorDescriptor::class, JsonlProvider()) + this.register(DoubleVectorDescriptor::class, JsonlProvider()) + + /* Struct descriptor. */ + this.register(LabelDescriptor::class, JsonlProvider()) + this.register(FileSourceMetadataDescriptor::class, JsonlProvider()) + this.register(VideoSourceMetadataDescriptor::class, JsonlProvider()) + this.register(TemporalMetadataDescriptor::class, JsonlProvider()) + this.register(Rectangle2DMetadataDescriptor::class, JsonlProvider()) + this.register(MediaDimensionsDescriptor::class, JsonlProvider()) + this.register(MapStructDescriptor::class, JsonlProvider()) + } + +} \ No newline at end of file diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlDescriptorProvider.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlDescriptorProvider.kt new file mode 100644 index 00000000..28dc5ea1 --- /dev/null +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlDescriptorProvider.kt @@ -0,0 +1,20 @@ +package org.vitrivr.engine.database.jsonl + +import org.vitrivr.engine.core.database.Connection +import org.vitrivr.engine.core.database.descriptor.DescriptorInitializer +import org.vitrivr.engine.core.database.descriptor.DescriptorProvider +import org.vitrivr.engine.core.database.descriptor.DescriptorReader +import org.vitrivr.engine.core.database.descriptor.DescriptorWriter +import org.vitrivr.engine.core.model.descriptor.Descriptor +import org.vitrivr.engine.core.model.metamodel.Schema + +class JsonlDescriptorProvider : DescriptorProvider { + override fun newInitializer( + connection: Connection, + field: Schema.Field<*, D> + ): DescriptorInitializer = JsonlInitializer(field, connection as JsonlConnection) + + override fun newReader(connection: Connection, field: Schema.Field<*, D>): DescriptorReader = JsonlReader(field, connection as JsonlConnection) + + override fun newWriter(connection: Connection, field: Schema.Field<*, D>): DescriptorWriter = JsonlWriter(field, connection as JsonlConnection) +} \ No newline at end of file diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlInitializer.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlInitializer.kt new file mode 100644 index 00000000..d8364c8b --- /dev/null +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlInitializer.kt @@ -0,0 +1,39 @@ +package org.vitrivr.engine.database.jsonl + +import org.vitrivr.engine.core.database.descriptor.DescriptorInitializer +import org.vitrivr.engine.core.model.descriptor.Descriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import java.io.IOException + +class JsonlInitializer(override val field: Schema.Field<*, D>, private val connection: JsonlConnection) : DescriptorInitializer { + + private val file = connection.getFile(field) + + override fun initialize() { + try{ + file.parentFile.mkdirs() + file.createNewFile() + } catch (ioe: IOException) { + LOGGER.error(ioe) { "Cannot initialize '${file.absolutePath}'" } + } catch (se: SecurityException) { + LOGGER.error(se) { "Cannot initialize '${file.absolutePath}'" } + } + + } + + override fun deinitialize() { + /* nop */ + } + + override fun isInitialized(): Boolean = file.exists() + + override fun truncate() { + try{ + file.delete() + } catch (ioe: IOException) { + LOGGER.error(ioe) { "Cannot delete '${file.absolutePath}'" } + } catch (se: SecurityException) { + LOGGER.error(se) { "Cannot delete '${file.absolutePath}'" } + } + } +} \ No newline at end of file diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlProvider.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlProvider.kt new file mode 100644 index 00000000..80a401c5 --- /dev/null +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlProvider.kt @@ -0,0 +1,24 @@ +package org.vitrivr.engine.database.jsonl + +import org.vitrivr.engine.core.database.Connection +import org.vitrivr.engine.core.database.descriptor.DescriptorInitializer +import org.vitrivr.engine.core.database.descriptor.DescriptorProvider +import org.vitrivr.engine.core.database.descriptor.DescriptorReader +import org.vitrivr.engine.core.database.descriptor.DescriptorWriter +import org.vitrivr.engine.core.model.descriptor.Descriptor +import org.vitrivr.engine.core.model.metamodel.Schema + +class JsonlProvider : DescriptorProvider { + + override fun newInitializer(connection: Connection, field: Schema.Field<*, D>): DescriptorInitializer { + TODO("Not yet implemented") + } + + override fun newReader(connection: Connection, field: Schema.Field<*, D>): DescriptorReader { + TODO("Not yet implemented") + } + + override fun newWriter(connection: Connection, field: Schema.Field<*, D>): DescriptorWriter { + TODO("Not yet implemented") + } +} \ No newline at end of file diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlReader.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlReader.kt new file mode 100644 index 00000000..611dd8aa --- /dev/null +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlReader.kt @@ -0,0 +1,48 @@ +package org.vitrivr.engine.database.jsonl + +import org.vitrivr.engine.core.database.descriptor.DescriptorReader +import org.vitrivr.engine.core.model.descriptor.Descriptor +import org.vitrivr.engine.core.model.descriptor.DescriptorId +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.query.Query +import org.vitrivr.engine.core.model.retrievable.RetrievableId +import org.vitrivr.engine.core.model.retrievable.Retrieved + +class JsonlReader(override val field: Schema.Field<*, D>, override val connection: JsonlConnection) : + DescriptorReader { + override fun exists(descriptorId: DescriptorId): Boolean { + TODO("Not yet implemented") + } + + override fun get(descriptorId: DescriptorId): D? { + TODO("Not yet implemented") + } + + override fun getAll(descriptorIds: Iterable): Sequence { + TODO("Not yet implemented") + } + + override fun getAll(): Sequence { + TODO("Not yet implemented") + } + + override fun getForRetrievable(retrievableId: RetrievableId): Sequence { + TODO("Not yet implemented") + } + + override fun getAllForRetrievable(retrievableIds: Iterable): Sequence { + TODO("Not yet implemented") + } + + override fun query(query: Query): Sequence { + TODO("Not yet implemented") + } + + override fun queryAndJoin(query: Query): Sequence { + TODO("Not yet implemented") + } + + override fun count(): Long { + TODO("Not yet implemented") + } +} \ No newline at end of file diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlRetrievableInitializer.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlRetrievableInitializer.kt new file mode 100644 index 00000000..2b3617be --- /dev/null +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlRetrievableInitializer.kt @@ -0,0 +1,22 @@ +package org.vitrivr.engine.database.jsonl + +import org.vitrivr.engine.core.database.retrievable.RetrievableInitializer + +class JsonlRetrievableInitializer(private val connection: JsonlConnection) : RetrievableInitializer { + + override fun initialize() { + TODO("Not yet implemented") + } + + override fun deinitialize() { + TODO("Not yet implemented") + } + + override fun isInitialized(): Boolean { + TODO("Not yet implemented") + } + + override fun truncate() { + TODO("Not yet implemented") + } +} \ No newline at end of file diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlRetrievableReader.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlRetrievableReader.kt new file mode 100644 index 00000000..e520cfa5 --- /dev/null +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlRetrievableReader.kt @@ -0,0 +1,41 @@ +package org.vitrivr.engine.database.jsonl + +import org.vitrivr.engine.core.database.Connection +import org.vitrivr.engine.core.database.retrievable.RetrievableReader +import org.vitrivr.engine.core.model.retrievable.Retrievable +import org.vitrivr.engine.core.model.retrievable.RetrievableId + +class JsonlRetrievableReader(override val connection: JsonlConnection) : RetrievableReader { + + override fun get(id: RetrievableId): Retrievable? { + TODO("Not yet implemented") + } + + override fun exists(id: RetrievableId): Boolean { + TODO("Not yet implemented") + } + + override fun getAll(ids: Iterable): Sequence { + TODO("Not yet implemented") + } + + override fun getConnections( + subjectIds: Collection, + predicates: Collection, + objectIds: Collection + ): Sequence> { + TODO("Not yet implemented") + } + + override fun getAll(): Sequence { + TODO("Not yet implemented") + } + + override fun count(): Long { + TODO("Not yet implemented") + } + + fun close() { + + } +} \ No newline at end of file diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlRetrievableWriter.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlRetrievableWriter.kt new file mode 100644 index 00000000..32cf0f68 --- /dev/null +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlRetrievableWriter.kt @@ -0,0 +1,55 @@ +package org.vitrivr.engine.database.jsonl + +import org.vitrivr.engine.core.database.retrievable.RetrievableWriter +import org.vitrivr.engine.core.model.relationship.Relationship +import org.vitrivr.engine.core.model.retrievable.Retrievable + +class JsonlRetrievableWriter(override val connection: JsonlConnection) : RetrievableWriter { + + override fun connect(relationship: Relationship): Boolean { + TODO("Not yet implemented") + } + + override fun connectAll(relationships: Iterable): Boolean { + relationships.forEach { connect(it) } + return true + } + + override fun disconnect(relationship: Relationship): Boolean { + LOGGER.warn { "JsonlRetrievableWriter.disconnect is not supported" } + return false + } + + override fun disconnectAll(relationships: Iterable): Boolean { + LOGGER.warn { "JsonlRetrievableWriter.disconnectAll is not supported" } + return false + } + + override fun add(item: Retrievable): Boolean { + TODO("Not yet implemented") + } + + override fun addAll(items: Iterable): Boolean { + items.forEach { add(it) } + return true + } + + override fun update(item: Retrievable): Boolean { + LOGGER.warn { "JsonlRetrievableWriter.update is not supported" } + return false + } + + override fun delete(item: Retrievable): Boolean { + LOGGER.warn { "JsonlRetrievableWriter.delete is not supported" } + return false + } + + override fun deleteAll(items: Iterable): Boolean { + LOGGER.warn { "JsonlRetrievableWriter.deleteAll is not supported" } + return false + } + + fun close() { + + } +} \ No newline at end of file diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlWriter.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlWriter.kt new file mode 100644 index 00000000..73cb6653 --- /dev/null +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlWriter.kt @@ -0,0 +1,71 @@ +package org.vitrivr.engine.database.jsonl + +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json +import org.vitrivr.engine.core.database.descriptor.DescriptorWriter +import org.vitrivr.engine.core.model.descriptor.AttributeName +import org.vitrivr.engine.core.model.descriptor.Descriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.types.Value +import org.vitrivr.engine.database.jsonl.JsonlConnection.Companion.DESCRIPTOR_ID_COLUMN_NAME +import org.vitrivr.engine.database.jsonl.JsonlConnection.Companion.RETRIEVABLE_ID_COLUMN_NAME +import java.io.FileWriter + + +class JsonlWriter(override val field: Schema.Field<*, D>, override val connection: JsonlConnection) : + DescriptorWriter { + + private val writer = FileWriter(connection.getFile(field), true) + + override fun add(item: D): Boolean { + + val valueMap = mutableMapOf?>( + DESCRIPTOR_ID_COLUMN_NAME to Value.UUIDValue(item.id), + RETRIEVABLE_ID_COLUMN_NAME to Value.UUIDValue( + item.retrievableId + ?: throw IllegalArgumentException("A struct descriptor must be associated with a retrievable ID.") + ) + ) + + + valueMap.putAll(item.values()) + + val list = AttributeContainerList( + item.layout().map { attribute -> + AttributeContainer( + attribute, + valueMap[attribute.name] + ) + } + ) + + + + writer.write(Json.encodeToString(list)) + writer.write("\n") + writer.flush() + + return true + + } + + override fun addAll(items: Iterable): Boolean { + items.forEach { add(it) } + return true + } + + override fun update(item: D): Boolean { + LOGGER.warn { "JsonlWriter.update is not supported" } + return false + } + + override fun delete(item: D): Boolean { + LOGGER.warn { "JsonlWriter.delete is not supported" } + return false + } + + override fun deleteAll(items: Iterable): Boolean { + LOGGER.warn { "JsonlWriter.deleteAll is not supported" } + return false + } +} \ No newline at end of file diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/ValueContainer.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/ValueContainer.kt new file mode 100644 index 00000000..e47276f0 --- /dev/null +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/ValueContainer.kt @@ -0,0 +1,117 @@ +package org.vitrivr.engine.database.jsonl + +import kotlinx.serialization.Serializable +import org.vitrivr.engine.core.model.serializer.DateSerializer +import org.vitrivr.engine.core.model.serializer.UUIDSerializer +import org.vitrivr.engine.core.model.types.Type +import org.vitrivr.engine.core.model.types.Value +import java.util.* + +@Serializable +sealed class ValueContainer(val type: Type) { + + companion object { + fun fromValue(value: Value<*>): ValueContainer = when (value) { + is Value.Boolean -> BooleanValueContainer(value.value) + is Value.Byte -> ByteValueContainer(value.value) + is Value.DateTime -> DateTimeValueContainer(value.value) + is Value.Double -> DoubleValueContainer(value.value) + is Value.Float -> FloatValueContainer(value.value) + is Value.Int -> IntValueContainer(value.value) + is Value.Long -> LongValueContainer(value.value) + is Value.Short -> ShortValueContainer(value.value) + is Value.String -> StringValueContainer(value.value) + is Value.Text -> TextValueContainer(value.value) + is Value.UUIDValue -> UuidValueContainer(value.value) + is Value.BooleanVector -> BooleanVectorValueContainer(value.value) + is Value.DoubleVector -> DoubleVectorValueContainer(value.value) + is Value.FloatVector -> FloatVectorValueContainer(value.value) + is Value.IntVector -> IntVectorValueContainer(value.value) + is Value.LongVector -> LongVectorValueContainer(value.value) + } + } + + abstract fun toValue(): Value<*> + +} + +@Serializable +class BooleanValueContainer(private val value: Boolean) : ValueContainer(Type.Boolean) { + override fun toValue(): Value = Value.Boolean(value) +} + +@Serializable +class ByteValueContainer(private val value: Byte) : ValueContainer(Type.Byte) { + override fun toValue(): Value = Value.Byte(value) +} + +@Serializable +class DateTimeValueContainer(@Serializable(DateSerializer::class) private val value: Date) : + ValueContainer(Type.Datetime) { + override fun toValue(): Value = Value.DateTime(value) +} + +@Serializable +class DoubleValueContainer(private val value: Double) : ValueContainer(Type.Double) { + override fun toValue(): Value = Value.Double(value) +} + +@Serializable +class FloatValueContainer(private val value: Float) : ValueContainer(Type.Float) { + override fun toValue(): Value = Value.Float(value) +} + +@Serializable +class IntValueContainer(private val value: Int) : ValueContainer(Type.Int) { + override fun toValue(): Value = Value.Int(value) +} + +@Serializable +class LongValueContainer(private val value: Long) : ValueContainer(Type.Long) { + override fun toValue(): Value = Value.Long(value) +} + +@Serializable +class ShortValueContainer(private val value: Short) : ValueContainer(Type.Short) { + override fun toValue(): Value = Value.Short(value) +} + +@Serializable +class StringValueContainer(private val value: String) : ValueContainer(Type.String) { + override fun toValue(): Value = Value.String(value) +} + +@Serializable +class TextValueContainer(private val value: String) : ValueContainer(Type.Text) { + override fun toValue(): Value = Value.Text(value) +} + +@Serializable +class UuidValueContainer(@Serializable(UUIDSerializer::class) private val value: UUID) : ValueContainer(Type.UUID) { + override fun toValue(): Value = Value.UUIDValue(value) +} + +@Serializable +class BooleanVectorValueContainer(private val value: BooleanArray) : ValueContainer(Type.BooleanVector(value.size)) { + override fun toValue(): Value = Value.BooleanVector(value) +} + +@Serializable +class DoubleVectorValueContainer(private val value: DoubleArray) : ValueContainer(Type.DoubleVector(value.size)) { + override fun toValue(): Value = Value.DoubleVector(value) +} + +@Serializable +class FloatVectorValueContainer(private val value: FloatArray) : ValueContainer(Type.FloatVector(value.size)) { + override fun toValue(): Value = Value.FloatVector(value) +} + +@Serializable +class IntVectorValueContainer(private val value: IntArray) : ValueContainer(Type.IntVector(value.size)) { + override fun toValue(): Value = Value.IntVector(value) +} + +@Serializable +class LongVectorValueContainer(private val value: LongArray) : ValueContainer(Type.LongVector(value.size)) { + override fun toValue(): Value = Value.LongVector(value) +} \ No newline at end of file diff --git a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/Common.kt b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/Common.kt index 310ffae0..6f7a87ec 100644 --- a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/Common.kt +++ b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/Common.kt @@ -61,6 +61,7 @@ internal fun Type.toCottontailType(): Types<*> = when (this) { Type.Float -> Types.Float Type.Double -> Types.Double Type.Datetime -> Types.Date + Type.UUID -> Types.Uuid is Type.BooleanVector -> Types.BooleanVector(this.dimensions) is Type.DoubleVector -> Types.DoubleVector(this.dimensions) is Type.FloatVector -> Types.FloatVector(this.dimensions) @@ -127,6 +128,7 @@ internal fun Value<*>.toCottontailValue(): PublicValue = when (this) { is Value.String -> StringValue(this.value) is Value.Text -> StringValue(this.value) is Value.DateTime -> DateValue(this.value) + is Value.UUIDValue -> UuidValue(this.value) is Value.BooleanVector -> BooleanVectorValue(this.value) is Value.DoubleVector -> DoubleVectorValue(this.value) is Value.FloatVector -> FloatVectorValue(this.value) From 9468813086a04a11bfc99bdbc4de08c758c69364 Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Sat, 27 Jul 2024 19:50:03 +0200 Subject: [PATCH 02/11] Added read/write logic for descriptors --- ...nlProvider.kt => AbstractJsonlProvider.kt} | 15 +--- .../database/jsonl/AbstractJsonlReader.kt | 79 +++++++++++++++++++ .../engine/database/jsonl/JsonlConnection.kt | 3 + .../database/jsonl/JsonlConnectionProvider.kt | 39 ++++----- .../database/jsonl/JsonlDescriptorProvider.kt | 20 ----- .../engine/database/jsonl/JsonlReader.kt | 48 ----------- .../engine/database/jsonl/JsonlWriter.kt | 10 ++- .../jsonl/{ => model}/AttributeContainer.kt | 2 +- .../{ => model}/AttributeContainerList.kt | 2 +- .../jsonl/{ => model}/ValueContainer.kt | 2 +- .../JsonlRetrievableInitializer.kt | 3 +- .../JsonlRetrievableReader.kt | 4 +- .../JsonlRetrievableWriter.kt | 4 +- .../jsonl/scalar/ScalarJsonlProvider.kt | 12 +++ .../jsonl/scalar/ScalarJsonlReader.kt | 39 +++++++++ .../jsonl/struct/StructJsonlProvider.kt | 13 +++ .../jsonl/struct/StructJsonlReader.kt | 40 ++++++++++ .../jsonl/vector/VectorJsonlProvider.kt | 12 +++ .../jsonl/vector/VectorJsonlReader.kt | 56 +++++++++++++ .../struct/StructDescriptorReader.kt | 1 + 20 files changed, 296 insertions(+), 108 deletions(-) rename vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/{JsonlProvider.kt => AbstractJsonlProvider.kt} (55%) create mode 100644 vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/AbstractJsonlReader.kt delete mode 100644 vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlDescriptorProvider.kt delete mode 100644 vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlReader.kt rename vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/{ => model}/AttributeContainer.kt (89%) rename vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/{ => model}/AttributeContainerList.kt (72%) rename vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/{ => model}/ValueContainer.kt (98%) rename vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/{ => retrievable}/JsonlRetrievableInitializer.kt (81%) rename vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/{ => retrievable}/JsonlRetrievableReader.kt (90%) rename vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/{ => retrievable}/JsonlRetrievableWriter.kt (91%) create mode 100644 vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlProvider.kt create mode 100644 vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlReader.kt create mode 100644 vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlProvider.kt create mode 100644 vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlReader.kt create mode 100644 vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/vector/VectorJsonlProvider.kt create mode 100644 vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/vector/VectorJsonlReader.kt diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlProvider.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/AbstractJsonlProvider.kt similarity index 55% rename from vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlProvider.kt rename to vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/AbstractJsonlProvider.kt index 80a401c5..2c36f349 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlProvider.kt +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/AbstractJsonlProvider.kt @@ -3,22 +3,13 @@ package org.vitrivr.engine.database.jsonl import org.vitrivr.engine.core.database.Connection import org.vitrivr.engine.core.database.descriptor.DescriptorInitializer import org.vitrivr.engine.core.database.descriptor.DescriptorProvider -import org.vitrivr.engine.core.database.descriptor.DescriptorReader import org.vitrivr.engine.core.database.descriptor.DescriptorWriter import org.vitrivr.engine.core.model.descriptor.Descriptor import org.vitrivr.engine.core.model.metamodel.Schema -class JsonlProvider : DescriptorProvider { +abstract class AbstractJsonlProvider : DescriptorProvider { - override fun newInitializer(connection: Connection, field: Schema.Field<*, D>): DescriptorInitializer { - TODO("Not yet implemented") - } + override fun newInitializer(connection: Connection, field: Schema.Field<*, D>): DescriptorInitializer = JsonlInitializer(field, connection as JsonlConnection) - override fun newReader(connection: Connection, field: Schema.Field<*, D>): DescriptorReader { - TODO("Not yet implemented") - } - - override fun newWriter(connection: Connection, field: Schema.Field<*, D>): DescriptorWriter { - TODO("Not yet implemented") - } + override fun newWriter(connection: Connection, field: Schema.Field<*, D>): DescriptorWriter = JsonlWriter(field, connection as JsonlConnection) } \ No newline at end of file diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/AbstractJsonlReader.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/AbstractJsonlReader.kt new file mode 100644 index 00000000..2ca39199 --- /dev/null +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/AbstractJsonlReader.kt @@ -0,0 +1,79 @@ +package org.vitrivr.engine.database.jsonl + +import kotlinx.serialization.SerializationException +import kotlinx.serialization.json.Json +import org.vitrivr.engine.core.database.descriptor.DescriptorReader +import org.vitrivr.engine.core.model.descriptor.Descriptor +import org.vitrivr.engine.core.model.descriptor.DescriptorId +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.query.Query +import org.vitrivr.engine.core.model.retrievable.RetrievableId +import org.vitrivr.engine.core.model.retrievable.Retrieved +import org.vitrivr.engine.database.jsonl.model.AttributeContainerList +import java.io.BufferedReader +import java.io.FileReader + +abstract class AbstractJsonlReader( + final override val field: Schema.Field<*, D>, + final override val connection: JsonlConnection +) : + DescriptorReader { + + /** Prototype used to create new instances. */ + protected val prototype = this.field.analyser.prototype(this.field) + + private val file = connection.getFile(field) + + protected abstract fun toDescriptor(list: AttributeContainerList): D + + override fun exists(descriptorId: DescriptorId): Boolean { + return getAll().any { it.id == descriptorId } + } + + override fun get(descriptorId: DescriptorId): D? { + return getAll().firstOrNull { it.id == descriptorId } + } + + override fun getAll(descriptorIds: Iterable): Sequence { + val ids = descriptorIds.toSet() + return getAll().filter { ids.contains(it.id) } + } + + override fun getAll(): Sequence { + + return BufferedReader(FileReader(file)).lineSequence().mapNotNull { + try { + val list = Json.decodeFromString(it) + return@mapNotNull toDescriptor(list) + } catch (se: SerializationException) { + LOGGER.error(se) { "Error during deserialization" } + null + } catch (ie: IllegalArgumentException) { + LOGGER.error(ie) { "Error during deserialization" } + null + } + } + + } + + override fun getForRetrievable(retrievableId: RetrievableId): Sequence { + return getAll().filter { it.retrievableId == retrievableId} + } + + override fun getAllForRetrievable(retrievableIds: Iterable): Sequence { + val ids = retrievableIds.toSet() + return getAll().filter { ids.contains(it.retrievableId) } + } + + override fun query(query: Query): Sequence { + TODO("Not yet implemented") + } + + override fun queryAndJoin(query: Query): Sequence { + TODO("Not yet implemented") + } + + override fun count(): Long { + return getAll().count().toLong() + } +} \ No newline at end of file diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnection.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnection.kt index 2dfbc1c3..8b142be2 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnection.kt +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnection.kt @@ -6,6 +6,9 @@ import org.vitrivr.engine.core.database.retrievable.RetrievableInitializer import org.vitrivr.engine.core.database.retrievable.RetrievableReader import org.vitrivr.engine.core.database.retrievable.RetrievableWriter import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.database.jsonl.retrievable.JsonlRetrievableInitializer +import org.vitrivr.engine.database.jsonl.retrievable.JsonlRetrievableReader +import org.vitrivr.engine.database.jsonl.retrievable.JsonlRetrievableWriter import java.io.File internal val LOGGER = logger("org.vitrivr.engine.database.jsonl.JsonlConnection") diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnectionProvider.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnectionProvider.kt index 207332cf..89d13145 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnectionProvider.kt +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnectionProvider.kt @@ -11,6 +11,9 @@ import org.vitrivr.engine.core.model.descriptor.struct.metadata.TemporalMetadata import org.vitrivr.engine.core.model.descriptor.struct.metadata.source.FileSourceMetadataDescriptor import org.vitrivr.engine.core.model.descriptor.struct.metadata.source.VideoSourceMetadataDescriptor import org.vitrivr.engine.core.model.descriptor.vector.* +import org.vitrivr.engine.database.jsonl.scalar.ScalarJsonlProvider +import org.vitrivr.engine.database.jsonl.struct.StructJsonlProvider +import org.vitrivr.engine.database.jsonl.vector.VectorJsonlProvider import java.io.File @@ -29,28 +32,28 @@ class JsonlConnectionProvider() : AbstractConnectionProvider() { override fun initialize() { /* Scalar descriptors. */ - this.register(BooleanDescriptor::class, JsonlProvider()) - this.register(IntDescriptor::class, JsonlProvider()) - this.register(LongDescriptor::class, JsonlProvider()) - this.register(FloatDescriptor::class, JsonlProvider()) - this.register(DoubleDescriptor::class, JsonlProvider()) - this.register(StringDescriptor::class, JsonlProvider()) + this.register(BooleanDescriptor::class, ScalarJsonlProvider) + this.register(IntDescriptor::class, ScalarJsonlProvider) + this.register(LongDescriptor::class, ScalarJsonlProvider) + this.register(FloatDescriptor::class, ScalarJsonlProvider) + this.register(DoubleDescriptor::class, ScalarJsonlProvider) + this.register(StringDescriptor::class, ScalarJsonlProvider) /* Vector descriptors. */ - this.register(BooleanVectorDescriptor::class, JsonlProvider()) - this.register(IntVectorDescriptor::class, JsonlProvider()) - this.register(LongVectorDescriptor::class, JsonlProvider()) - this.register(FloatVectorDescriptor::class, JsonlProvider()) - this.register(DoubleVectorDescriptor::class, JsonlProvider()) + this.register(BooleanVectorDescriptor::class, VectorJsonlProvider) + this.register(IntVectorDescriptor::class, VectorJsonlProvider) + this.register(LongVectorDescriptor::class, VectorJsonlProvider) + this.register(FloatVectorDescriptor::class, VectorJsonlProvider) + this.register(DoubleVectorDescriptor::class, VectorJsonlProvider) /* Struct descriptor. */ - this.register(LabelDescriptor::class, JsonlProvider()) - this.register(FileSourceMetadataDescriptor::class, JsonlProvider()) - this.register(VideoSourceMetadataDescriptor::class, JsonlProvider()) - this.register(TemporalMetadataDescriptor::class, JsonlProvider()) - this.register(Rectangle2DMetadataDescriptor::class, JsonlProvider()) - this.register(MediaDimensionsDescriptor::class, JsonlProvider()) - this.register(MapStructDescriptor::class, JsonlProvider()) + this.register(LabelDescriptor::class, StructJsonlProvider) + this.register(FileSourceMetadataDescriptor::class, StructJsonlProvider) + this.register(VideoSourceMetadataDescriptor::class, StructJsonlProvider) + this.register(TemporalMetadataDescriptor::class, StructJsonlProvider) + this.register(Rectangle2DMetadataDescriptor::class, StructJsonlProvider) + this.register(MediaDimensionsDescriptor::class, StructJsonlProvider) + this.register(MapStructDescriptor::class, StructJsonlProvider) } } \ No newline at end of file diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlDescriptorProvider.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlDescriptorProvider.kt deleted file mode 100644 index 28dc5ea1..00000000 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlDescriptorProvider.kt +++ /dev/null @@ -1,20 +0,0 @@ -package org.vitrivr.engine.database.jsonl - -import org.vitrivr.engine.core.database.Connection -import org.vitrivr.engine.core.database.descriptor.DescriptorInitializer -import org.vitrivr.engine.core.database.descriptor.DescriptorProvider -import org.vitrivr.engine.core.database.descriptor.DescriptorReader -import org.vitrivr.engine.core.database.descriptor.DescriptorWriter -import org.vitrivr.engine.core.model.descriptor.Descriptor -import org.vitrivr.engine.core.model.metamodel.Schema - -class JsonlDescriptorProvider : DescriptorProvider { - override fun newInitializer( - connection: Connection, - field: Schema.Field<*, D> - ): DescriptorInitializer = JsonlInitializer(field, connection as JsonlConnection) - - override fun newReader(connection: Connection, field: Schema.Field<*, D>): DescriptorReader = JsonlReader(field, connection as JsonlConnection) - - override fun newWriter(connection: Connection, field: Schema.Field<*, D>): DescriptorWriter = JsonlWriter(field, connection as JsonlConnection) -} \ No newline at end of file diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlReader.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlReader.kt deleted file mode 100644 index 611dd8aa..00000000 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlReader.kt +++ /dev/null @@ -1,48 +0,0 @@ -package org.vitrivr.engine.database.jsonl - -import org.vitrivr.engine.core.database.descriptor.DescriptorReader -import org.vitrivr.engine.core.model.descriptor.Descriptor -import org.vitrivr.engine.core.model.descriptor.DescriptorId -import org.vitrivr.engine.core.model.metamodel.Schema -import org.vitrivr.engine.core.model.query.Query -import org.vitrivr.engine.core.model.retrievable.RetrievableId -import org.vitrivr.engine.core.model.retrievable.Retrieved - -class JsonlReader(override val field: Schema.Field<*, D>, override val connection: JsonlConnection) : - DescriptorReader { - override fun exists(descriptorId: DescriptorId): Boolean { - TODO("Not yet implemented") - } - - override fun get(descriptorId: DescriptorId): D? { - TODO("Not yet implemented") - } - - override fun getAll(descriptorIds: Iterable): Sequence { - TODO("Not yet implemented") - } - - override fun getAll(): Sequence { - TODO("Not yet implemented") - } - - override fun getForRetrievable(retrievableId: RetrievableId): Sequence { - TODO("Not yet implemented") - } - - override fun getAllForRetrievable(retrievableIds: Iterable): Sequence { - TODO("Not yet implemented") - } - - override fun query(query: Query): Sequence { - TODO("Not yet implemented") - } - - override fun queryAndJoin(query: Query): Sequence { - TODO("Not yet implemented") - } - - override fun count(): Long { - TODO("Not yet implemented") - } -} \ No newline at end of file diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlWriter.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlWriter.kt index 73cb6653..16736878 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlWriter.kt +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlWriter.kt @@ -9,11 +9,13 @@ import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.types.Value import org.vitrivr.engine.database.jsonl.JsonlConnection.Companion.DESCRIPTOR_ID_COLUMN_NAME import org.vitrivr.engine.database.jsonl.JsonlConnection.Companion.RETRIEVABLE_ID_COLUMN_NAME +import org.vitrivr.engine.database.jsonl.model.AttributeContainer +import org.vitrivr.engine.database.jsonl.model.AttributeContainerList import java.io.FileWriter class JsonlWriter(override val field: Schema.Field<*, D>, override val connection: JsonlConnection) : - DescriptorWriter { + DescriptorWriter, AutoCloseable { private val writer = FileWriter(connection.getFile(field), true) @@ -27,7 +29,6 @@ class JsonlWriter(override val field: Schema.Field<*, D>, overri ) ) - valueMap.putAll(item.values()) val list = AttributeContainerList( @@ -40,7 +41,6 @@ class JsonlWriter(override val field: Schema.Field<*, D>, overri ) - writer.write(Json.encodeToString(list)) writer.write("\n") writer.flush() @@ -68,4 +68,8 @@ class JsonlWriter(override val field: Schema.Field<*, D>, overri LOGGER.warn { "JsonlWriter.deleteAll is not supported" } return false } + + override fun close() { + writer.close() + } } \ No newline at end of file diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/AttributeContainer.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/AttributeContainer.kt similarity index 89% rename from vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/AttributeContainer.kt rename to vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/AttributeContainer.kt index 727d0e96..9c83cbd4 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/AttributeContainer.kt +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/AttributeContainer.kt @@ -1,4 +1,4 @@ -package org.vitrivr.engine.database.jsonl +package org.vitrivr.engine.database.jsonl.model import kotlinx.serialization.Serializable import org.vitrivr.engine.core.model.descriptor.Attribute diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/AttributeContainerList.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/AttributeContainerList.kt similarity index 72% rename from vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/AttributeContainerList.kt rename to vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/AttributeContainerList.kt index 2d1b12a4..ae226f9a 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/AttributeContainerList.kt +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/AttributeContainerList.kt @@ -1,4 +1,4 @@ -package org.vitrivr.engine.database.jsonl +package org.vitrivr.engine.database.jsonl.model import kotlinx.serialization.Serializable diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/ValueContainer.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/ValueContainer.kt similarity index 98% rename from vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/ValueContainer.kt rename to vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/ValueContainer.kt index e47276f0..f66b9377 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/ValueContainer.kt +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/ValueContainer.kt @@ -1,4 +1,4 @@ -package org.vitrivr.engine.database.jsonl +package org.vitrivr.engine.database.jsonl.model import kotlinx.serialization.Serializable import org.vitrivr.engine.core.model.serializer.DateSerializer diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlRetrievableInitializer.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableInitializer.kt similarity index 81% rename from vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlRetrievableInitializer.kt rename to vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableInitializer.kt index 2b3617be..ee0bbc27 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlRetrievableInitializer.kt +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableInitializer.kt @@ -1,6 +1,7 @@ -package org.vitrivr.engine.database.jsonl +package org.vitrivr.engine.database.jsonl.retrievable import org.vitrivr.engine.core.database.retrievable.RetrievableInitializer +import org.vitrivr.engine.database.jsonl.JsonlConnection class JsonlRetrievableInitializer(private val connection: JsonlConnection) : RetrievableInitializer { diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlRetrievableReader.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableReader.kt similarity index 90% rename from vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlRetrievableReader.kt rename to vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableReader.kt index e520cfa5..1dcc467c 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlRetrievableReader.kt +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableReader.kt @@ -1,9 +1,9 @@ -package org.vitrivr.engine.database.jsonl +package org.vitrivr.engine.database.jsonl.retrievable -import org.vitrivr.engine.core.database.Connection import org.vitrivr.engine.core.database.retrievable.RetrievableReader import org.vitrivr.engine.core.model.retrievable.Retrievable import org.vitrivr.engine.core.model.retrievable.RetrievableId +import org.vitrivr.engine.database.jsonl.JsonlConnection class JsonlRetrievableReader(override val connection: JsonlConnection) : RetrievableReader { diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlRetrievableWriter.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableWriter.kt similarity index 91% rename from vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlRetrievableWriter.kt rename to vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableWriter.kt index 32cf0f68..6c10f15c 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlRetrievableWriter.kt +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableWriter.kt @@ -1,8 +1,10 @@ -package org.vitrivr.engine.database.jsonl +package org.vitrivr.engine.database.jsonl.retrievable import org.vitrivr.engine.core.database.retrievable.RetrievableWriter import org.vitrivr.engine.core.model.relationship.Relationship import org.vitrivr.engine.core.model.retrievable.Retrievable +import org.vitrivr.engine.database.jsonl.JsonlConnection +import org.vitrivr.engine.database.jsonl.LOGGER class JsonlRetrievableWriter(override val connection: JsonlConnection) : RetrievableWriter { diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlProvider.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlProvider.kt new file mode 100644 index 00000000..5e6e9b95 --- /dev/null +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlProvider.kt @@ -0,0 +1,12 @@ +package org.vitrivr.engine.database.jsonl.scalar + +import org.vitrivr.engine.core.database.Connection +import org.vitrivr.engine.core.database.descriptor.DescriptorReader +import org.vitrivr.engine.core.model.descriptor.scalar.ScalarDescriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.database.jsonl.AbstractJsonlProvider +import org.vitrivr.engine.database.jsonl.JsonlConnection + +object ScalarJsonlProvider: AbstractJsonlProvider>() { + override fun newReader(connection: Connection, field: Schema.Field<*, ScalarDescriptor<*>>): DescriptorReader> = ScalarJsonlReader(field, connection as JsonlConnection) +} \ No newline at end of file diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlReader.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlReader.kt new file mode 100644 index 00000000..caf1a50c --- /dev/null +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlReader.kt @@ -0,0 +1,39 @@ +package org.vitrivr.engine.database.jsonl.scalar + +import org.vitrivr.engine.core.model.descriptor.scalar.* +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.types.Value +import org.vitrivr.engine.database.jsonl.AbstractJsonlReader +import org.vitrivr.engine.database.jsonl.model.AttributeContainerList +import org.vitrivr.engine.database.jsonl.JsonlConnection +import org.vitrivr.engine.database.jsonl.JsonlConnection.Companion.DESCRIPTOR_ID_COLUMN_NAME +import org.vitrivr.engine.database.jsonl.JsonlConnection.Companion.RETRIEVABLE_ID_COLUMN_NAME + +class ScalarJsonlReader( + field: Schema.Field<*, ScalarDescriptor<*>>, + connection: JsonlConnection +) : AbstractJsonlReader>(field, connection) { + + override fun toDescriptor(list: AttributeContainerList): ScalarDescriptor<*> { + + val map = list.list.associateBy { it.attribute.name } + val retrievableId = (map[DESCRIPTOR_ID_COLUMN_NAME]?.value!!.toValue() as Value.UUIDValue).value + val descriptorId = (map[RETRIEVABLE_ID_COLUMN_NAME]?.value!!.toValue() as Value.UUIDValue).value + val value = map["value"]?.value!!.toValue() + + return when(prototype) { + is BooleanDescriptor -> BooleanDescriptor(retrievableId, descriptorId, value as Value.Boolean) + is DoubleDescriptor -> DoubleDescriptor(retrievableId, descriptorId, value as Value.Double) + is FloatDescriptor -> FloatDescriptor(retrievableId, descriptorId, value as Value.Float) + is IntDescriptor -> IntDescriptor(retrievableId, descriptorId, value as Value.Int) + is LongDescriptor -> LongDescriptor(retrievableId, descriptorId, value as Value.Long) + is StringDescriptor -> StringDescriptor(retrievableId, descriptorId, value as Value.String) + else -> { + error("Unsupported type $prototype") + } + } + + + } + +} \ No newline at end of file diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlProvider.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlProvider.kt new file mode 100644 index 00000000..9c4507c6 --- /dev/null +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlProvider.kt @@ -0,0 +1,13 @@ +package org.vitrivr.engine.database.jsonl.struct + +import org.vitrivr.engine.core.database.Connection +import org.vitrivr.engine.core.database.descriptor.DescriptorReader +import org.vitrivr.engine.core.model.descriptor.struct.StructDescriptor +import org.vitrivr.engine.core.model.descriptor.vector.VectorDescriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.database.jsonl.AbstractJsonlProvider +import org.vitrivr.engine.database.jsonl.JsonlConnection + +object StructJsonlProvider: AbstractJsonlProvider() { + override fun newReader(connection: Connection, field: Schema.Field<*, StructDescriptor>): DescriptorReader = StructJsonlReader(field, connection as JsonlConnection) +} \ No newline at end of file diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlReader.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlReader.kt new file mode 100644 index 00000000..e7a6785d --- /dev/null +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlReader.kt @@ -0,0 +1,40 @@ +package org.vitrivr.engine.database.jsonl.struct + +import org.vitrivr.engine.core.model.descriptor.AttributeName +import org.vitrivr.engine.core.model.descriptor.struct.StructDescriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.types.Value +import org.vitrivr.engine.database.jsonl.AbstractJsonlReader +import org.vitrivr.engine.database.jsonl.JsonlConnection +import org.vitrivr.engine.database.jsonl.JsonlConnection.Companion.DESCRIPTOR_ID_COLUMN_NAME +import org.vitrivr.engine.database.jsonl.JsonlConnection.Companion.RETRIEVABLE_ID_COLUMN_NAME +import org.vitrivr.engine.database.jsonl.model.AttributeContainerList +import kotlin.reflect.full.primaryConstructor + +class StructJsonlReader ( + field: Schema.Field<*, StructDescriptor>, + connection: JsonlConnection +) : AbstractJsonlReader(field, connection) { + + override fun toDescriptor(list: AttributeContainerList): StructDescriptor { + + val map = list.list.associateBy { it.attribute.name } + val constructor = this.field.analyser.descriptorClass.primaryConstructor ?: throw IllegalStateException("Provided type ${this.field.analyser.descriptorClass} does not have a primary constructor.") + val valueMap = mutableMapOf>() + + val retrievableId = (map[DESCRIPTOR_ID_COLUMN_NAME]?.value!!.toValue() as Value.UUIDValue).value + val descriptorId = (map[RETRIEVABLE_ID_COLUMN_NAME]?.value!!.toValue() as Value.UUIDValue).value + val parameters: MutableList = mutableListOf( + descriptorId, + retrievableId, + valueMap + ) + + prototype.layout().forEach { attribute -> + val value = map[attribute.name]!!.value?.toValue()!! + valueMap[attribute.name] = value + } + + return constructor.call(*parameters.toTypedArray()) + } +} \ No newline at end of file diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/vector/VectorJsonlProvider.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/vector/VectorJsonlProvider.kt new file mode 100644 index 00000000..4cb88c3c --- /dev/null +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/vector/VectorJsonlProvider.kt @@ -0,0 +1,12 @@ +package org.vitrivr.engine.database.jsonl.vector + +import org.vitrivr.engine.core.database.Connection +import org.vitrivr.engine.core.database.descriptor.DescriptorReader +import org.vitrivr.engine.core.model.descriptor.vector.VectorDescriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.database.jsonl.AbstractJsonlProvider +import org.vitrivr.engine.database.jsonl.JsonlConnection + +object VectorJsonlProvider: AbstractJsonlProvider>() { + override fun newReader(connection: Connection, field: Schema.Field<*, VectorDescriptor<*>>): DescriptorReader> = VectorJsonlReader(field, connection as JsonlConnection) +} \ No newline at end of file diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/vector/VectorJsonlReader.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/vector/VectorJsonlReader.kt new file mode 100644 index 00000000..cb1420e0 --- /dev/null +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/vector/VectorJsonlReader.kt @@ -0,0 +1,56 @@ +package org.vitrivr.engine.database.jsonl.vector + +import org.vitrivr.engine.core.model.descriptor.vector.* +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.types.Value +import org.vitrivr.engine.database.jsonl.AbstractJsonlReader +import org.vitrivr.engine.database.jsonl.model.AttributeContainerList +import org.vitrivr.engine.database.jsonl.JsonlConnection +import org.vitrivr.engine.database.jsonl.JsonlConnection.Companion.DESCRIPTOR_ID_COLUMN_NAME +import org.vitrivr.engine.database.jsonl.JsonlConnection.Companion.RETRIEVABLE_ID_COLUMN_NAME + +class VectorJsonlReader( + field: Schema.Field<*, VectorDescriptor<*>>, + connection: JsonlConnection +) : AbstractJsonlReader>(field, connection) { + + override fun toDescriptor(list: AttributeContainerList): VectorDescriptor<*> { + + val map = list.list.associateBy { it.attribute.name } + val retrievableId = (map[DESCRIPTOR_ID_COLUMN_NAME]?.value!!.toValue() as Value.UUIDValue).value + val descriptorId = (map[RETRIEVABLE_ID_COLUMN_NAME]?.value!!.toValue() as Value.UUIDValue).value + val value = map["vector"]?.value!!.toValue() + + return when (prototype) { + is BooleanVectorDescriptor -> BooleanVectorDescriptor( + descriptorId, + retrievableId, + value as Value.BooleanVector + ) + + is FloatVectorDescriptor -> FloatVectorDescriptor( + descriptorId, + retrievableId, + value as Value.FloatVector + ) + + is DoubleVectorDescriptor -> DoubleVectorDescriptor( + descriptorId, + retrievableId, + value as Value.DoubleVector + ) + + is IntVectorDescriptor -> IntVectorDescriptor( + descriptorId, + retrievableId, + value as Value.IntVector + ) + + is LongVectorDescriptor -> LongVectorDescriptor( + descriptorId, + retrievableId, + value as Value.LongVector + ) + } + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/struct/StructDescriptorReader.kt b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/struct/StructDescriptorReader.kt index c0df79ec..8a2611a1 100644 --- a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/struct/StructDescriptorReader.kt +++ b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/struct/StructDescriptorReader.kt @@ -74,6 +74,7 @@ class StructDescriptorReader(field: Schema.Field<*, StructDescriptor>, connectio Types.Long -> tuple.asLong(name)?.let { Value.Long(it) } Types.Short -> tuple.asShort(name)?.let { Value.Short(it) } Types.String -> tuple.asString(name)?.let { Value.String(it) } + Types.Uuid -> tuple.asUuidValue(name)?.let { Value.UUIDValue(it.value) } is Types.BooleanVector -> tuple.asBooleanVector(name)?.let { Value.BooleanVector(it) } is Types.DoubleVector -> tuple.asDoubleVector(name)?.let { Value.DoubleVector(it) } is Types.FloatVector -> tuple.asFloatVector(name)?.let { Value.FloatVector(it) } From 90c7646a7538b790cb4434571f9ba6025a45bd5d Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Sat, 27 Jul 2024 21:16:33 +0200 Subject: [PATCH 03/11] Added read/write logic for retrievables --- .../engine/database/jsonl/JsonlConnection.kt | 24 +------- .../database/jsonl/model/JsonlRelationship.kt | 13 ++++ .../database/jsonl/model/JsonlRetrievable.kt | 13 ++++ .../JsonlRetrievableInitializer.kt | 27 +++++++- .../retrievable/JsonlRetrievableReader.kt | 61 ++++++++++++++----- .../retrievable/JsonlRetrievableWriter.kt | 28 +++++++-- ...vr.engine.core.database.ConnectionProvider | 1 + .../engine/database/pgvector/Utilities.kt | 1 + .../descriptor/PgDescriptorInitializer.kt | 1 + .../struct/StructDescriptorReader.kt | 1 + 10 files changed, 124 insertions(+), 46 deletions(-) create mode 100644 vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/JsonlRelationship.kt create mode 100644 vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/JsonlRetrievable.kt create mode 100644 vitrivr-engine-index/src/main/resources/META-INF/services/org.vitrivr.engine.core.database.ConnectionProvider diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnection.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnection.kt index 8b142be2..d8f6a1fc 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnection.kt +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnection.kt @@ -24,7 +24,7 @@ class JsonlConnection( return action(Unit) } - private val schemaRoot = File(root, schemaName) + internal val schemaRoot = File(root, schemaName) fun getFile(field: Schema.Field<*, *>) = File(schemaRoot, "${field.fieldName}.jsonl") @@ -42,35 +42,13 @@ class JsonlConnection( override fun close() { writer.close() - reader.close() } companion object { - /** The name of the retrievable entity. */ - const val RETRIEVABLE_ENTITY_NAME = "retrievable" - /** The column name of a retrievable ID. */ const val RETRIEVABLE_ID_COLUMN_NAME = "retrievableId" - /** The column name of a retrievable ID. */ - const val RETRIEVABLE_TYPE_COLUMN_NAME = "type" - - /** The name of the retrievable entity. */ - const val RELATIONSHIP_ENTITY_NAME = "relationships" - - /** The column name of a retrievable ID. */ - const val SUBJECT_ID_COLUMN_NAME = "subjectId" - - /** The column name of a retrievable ID. */ - const val OBJECT_ID_COLUMN_NAME = "objectId" - - /** The column name of a retrievable ID. */ - const val PREDICATE_COLUMN_NAME = "predicate" - - /** The prefix for descriptor entities. */ - const val DESCRIPTOR_ENTITY_PREFIX = "descriptor" - /** The column name of a descriptor ID. */ const val DESCRIPTOR_ID_COLUMN_NAME = "descriptorId" } diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/JsonlRelationship.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/JsonlRelationship.kt new file mode 100644 index 00000000..efffd87e --- /dev/null +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/JsonlRelationship.kt @@ -0,0 +1,13 @@ +package org.vitrivr.engine.database.jsonl.model + +import kotlinx.serialization.Serializable +import org.vitrivr.engine.core.model.relationship.Relationship +import org.vitrivr.engine.core.model.retrievable.RetrievableId + +@Serializable +data class JsonlRelationship(val sub: RetrievableId, val pred: String, val obj: RetrievableId) { + constructor(relationship: Relationship) : this(relationship.subjectId, relationship.predicate, relationship.objectId) + + fun toTriple() : Triple = Triple(sub, pred, obj) + +} diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/JsonlRetrievable.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/JsonlRetrievable.kt new file mode 100644 index 00000000..0e9ac6d0 --- /dev/null +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/JsonlRetrievable.kt @@ -0,0 +1,13 @@ +package org.vitrivr.engine.database.jsonl.model + +import kotlinx.serialization.Serializable +import org.vitrivr.engine.core.model.retrievable.Retrievable +import org.vitrivr.engine.core.model.retrievable.RetrievableId +import org.vitrivr.engine.core.model.retrievable.Retrieved + +@Serializable +data class JsonlRetrievable(val id: RetrievableId, val type: String) { + fun toRetrieved(): Retrieved = Retrieved(id, if (type.isEmpty()) null else type, false) + + constructor(retrievable: Retrievable) : this(retrievable.id, retrievable.type ?: "") +} diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableInitializer.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableInitializer.kt index ee0bbc27..2ff51baa 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableInitializer.kt +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableInitializer.kt @@ -2,15 +2,29 @@ package org.vitrivr.engine.database.jsonl.retrievable import org.vitrivr.engine.core.database.retrievable.RetrievableInitializer import org.vitrivr.engine.database.jsonl.JsonlConnection +import org.vitrivr.engine.database.jsonl.LOGGER +import java.io.File +import java.io.IOException class JsonlRetrievableInitializer(private val connection: JsonlConnection) : RetrievableInitializer { + private val retrievableFile = File(connection.schemaRoot, "retrievables.jsonl") + private val connectionFile = File(connection.schemaRoot, "retrievable_connections.jsonl") + override fun initialize() { - TODO("Not yet implemented") + try { + connection.schemaRoot.mkdirs() + retrievableFile.createNewFile() + connectionFile.createNewFile() + } catch (ioe: IOException) { + LOGGER.error(ioe) { "Cannot initialize '${connection.schemaRoot.absolutePath}'" } + } catch (se: SecurityException) { + LOGGER.error(se) { "Cannot initialize '${connection.schemaRoot.absolutePath}'" } + } } override fun deinitialize() { - TODO("Not yet implemented") + /* nop */ } override fun isInitialized(): Boolean { @@ -18,6 +32,13 @@ class JsonlRetrievableInitializer(private val connection: JsonlConnection) : Ret } override fun truncate() { - TODO("Not yet implemented") + try { + retrievableFile.delete() + connectionFile.delete() + } catch (ioe: IOException) { + LOGGER.error(ioe) { "Cannot delete '${connection.schemaRoot.absolutePath}'" } + } catch (se: SecurityException) { + LOGGER.error(se) { "Cannot delete '${connection.schemaRoot.absolutePath}'" } + } } } \ No newline at end of file diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableReader.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableReader.kt index 1dcc467c..5da3a293 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableReader.kt +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableReader.kt @@ -1,22 +1,30 @@ package org.vitrivr.engine.database.jsonl.retrievable +import kotlinx.serialization.SerializationException +import kotlinx.serialization.json.Json import org.vitrivr.engine.core.database.retrievable.RetrievableReader import org.vitrivr.engine.core.model.retrievable.Retrievable import org.vitrivr.engine.core.model.retrievable.RetrievableId import org.vitrivr.engine.database.jsonl.JsonlConnection +import org.vitrivr.engine.database.jsonl.LOGGER +import org.vitrivr.engine.database.jsonl.model.JsonlRelationship +import org.vitrivr.engine.database.jsonl.model.JsonlRetrievable +import java.io.BufferedReader +import java.io.File +import java.io.FileReader class JsonlRetrievableReader(override val connection: JsonlConnection) : RetrievableReader { - override fun get(id: RetrievableId): Retrievable? { - TODO("Not yet implemented") - } + private val retrievableFile = File(connection.schemaRoot, "retrievables.jsonl") + private val connectionFile = File(connection.schemaRoot, "retrievable_connections.jsonl") - override fun exists(id: RetrievableId): Boolean { - TODO("Not yet implemented") - } + override fun get(id: RetrievableId): Retrievable? = getAll().firstOrNull { it.id == id } + + override fun exists(id: RetrievableId): Boolean = get(id) != null override fun getAll(ids: Iterable): Sequence { - TODO("Not yet implemented") + val idSet = ids.toSet() + return getAll().filter { idSet.contains(it.id) } } override fun getConnections( @@ -24,18 +32,41 @@ class JsonlRetrievableReader(override val connection: JsonlConnection) : Retriev predicates: Collection, objectIds: Collection ): Sequence> { - TODO("Not yet implemented") - } + val subIds = subjectIds.toSet() + val predIds = predicates.toSet() + val objIds = objectIds.toSet() - override fun getAll(): Sequence { - TODO("Not yet implemented") + return BufferedReader(FileReader(connectionFile)).lineSequence().mapNotNull { + try { + Json.decodeFromString(it) + } catch (se: SerializationException) { + LOGGER.error(se) { "Error during deserialization" } + null + } catch (ie: IllegalArgumentException) { + LOGGER.error(ie) { "Error during deserialization" } + null + } + }.filter { + (subIds.isEmpty() || subIds.contains(it.sub)) && + (predIds.isEmpty() || predIds.contains(it.pred)) && + (objIds.isEmpty() || objIds.contains(it.obj)) + }.map { it.toTriple() } } - override fun count(): Long { - TODO("Not yet implemented") + override fun getAll(): Sequence { + return BufferedReader(FileReader(retrievableFile)).lineSequence().mapNotNull { + try { + Json.decodeFromString(it).toRetrieved() + } catch (se: SerializationException) { + LOGGER.error(se) { "Error during deserialization" } + null + } catch (ie: IllegalArgumentException) { + LOGGER.error(ie) { "Error during deserialization" } + null + } + } } - fun close() { + override fun count(): Long = getAll().count().toLong() - } } \ No newline at end of file diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableWriter.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableWriter.kt index 6c10f15c..357e9cd2 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableWriter.kt +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableWriter.kt @@ -1,15 +1,28 @@ package org.vitrivr.engine.database.jsonl.retrievable +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json import org.vitrivr.engine.core.database.retrievable.RetrievableWriter import org.vitrivr.engine.core.model.relationship.Relationship import org.vitrivr.engine.core.model.retrievable.Retrievable import org.vitrivr.engine.database.jsonl.JsonlConnection import org.vitrivr.engine.database.jsonl.LOGGER +import org.vitrivr.engine.database.jsonl.model.JsonlRelationship +import org.vitrivr.engine.database.jsonl.model.JsonlRetrievable +import java.io.File +import java.io.FileWriter -class JsonlRetrievableWriter(override val connection: JsonlConnection) : RetrievableWriter { +class JsonlRetrievableWriter(override val connection: JsonlConnection) : RetrievableWriter, AutoCloseable { + private val retrievableWriter = FileWriter(File(connection.schemaRoot, "retrievables.jsonl"), true) + private val connectionWriter = FileWriter(File(connection.schemaRoot, "retrievable_connections.jsonl"), true) + + @Synchronized override fun connect(relationship: Relationship): Boolean { - TODO("Not yet implemented") + connectionWriter.write(Json.encodeToString(JsonlRelationship(relationship))) + connectionWriter.write("\n") + connectionWriter.flush() + return true } override fun connectAll(relationships: Iterable): Boolean { @@ -27,8 +40,12 @@ class JsonlRetrievableWriter(override val connection: JsonlConnection) : Retriev return false } + @Synchronized override fun add(item: Retrievable): Boolean { - TODO("Not yet implemented") + retrievableWriter.write(Json.encodeToString(JsonlRetrievable(item))) + retrievableWriter.write("\n") + retrievableWriter.flush() + return true } override fun addAll(items: Iterable): Boolean { @@ -51,7 +68,8 @@ class JsonlRetrievableWriter(override val connection: JsonlConnection) : Retriev return false } - fun close() { - + override fun close() { + retrievableWriter.close() + connectionWriter.close() } } \ No newline at end of file diff --git a/vitrivr-engine-index/src/main/resources/META-INF/services/org.vitrivr.engine.core.database.ConnectionProvider b/vitrivr-engine-index/src/main/resources/META-INF/services/org.vitrivr.engine.core.database.ConnectionProvider new file mode 100644 index 00000000..41cabd09 --- /dev/null +++ b/vitrivr-engine-index/src/main/resources/META-INF/services/org.vitrivr.engine.core.database.ConnectionProvider @@ -0,0 +1 @@ +org.vitrivr.engine.database.jsonl.JsonlConnectionProvider \ No newline at end of file diff --git a/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/Utilities.kt b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/Utilities.kt index 4c620286..8ff0258a 100644 --- a/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/Utilities.kt +++ b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/Utilities.kt @@ -50,6 +50,7 @@ internal fun Type.toSql(): Int = when (this) { Type.Datetime -> JDBCType.DATE Type.String -> JDBCType.VARCHAR Type.Text -> JDBCType.CLOB + Type.UUID -> JDBCType.OTHER is Type.BooleanVector -> JDBCType.ARRAY is Type.DoubleVector -> JDBCType.ARRAY is Type.FloatVector -> JDBCType.ARRAY diff --git a/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/PgDescriptorInitializer.kt b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/PgDescriptorInitializer.kt index 827a39a7..4e7879dd 100644 --- a/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/PgDescriptorInitializer.kt +++ b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/PgDescriptorInitializer.kt @@ -54,6 +54,7 @@ open class PgDescriptorInitializer(final override val field: Sch Type.Float -> statement.append("\"${field.name}\" real, ") Type.Double -> statement.append("\"${field.name}\" double precision, ") Type.Datetime -> statement.append("\"${field.name}\" datetime, ") + Type.UUID -> statement.append("\"${field.name}\" uuid, ") is Type.BooleanVector -> statement.append("\"${field.name}\" bit(${field.type.dimensions}), ") is Type.DoubleVector -> statement.append("\"${field.name}\" vector(${field.type.dimensions}), ") is Type.FloatVector -> statement.append("\"${field.name}\" vector(${field.type.dimensions}), ") diff --git a/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/struct/StructDescriptorReader.kt b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/struct/StructDescriptorReader.kt index d9870e62..7ac1461b 100644 --- a/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/struct/StructDescriptorReader.kt +++ b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/struct/StructDescriptorReader.kt @@ -65,6 +65,7 @@ class StructDescriptorReader(field: Schema.Field<*, StructDescriptor>, connectio Type.Float -> result.getFloat(field.name).let { Value.Float(it) } Type.Double -> result.getDouble(field.name).let { Value.Double(it) } Type.Datetime -> result.getDate(field.name).toInstant().let { Value.DateTime(Date(it.toEpochMilli())) } + Type.UUID -> result.getObject(field.name, UUID::class.java).let { Value.UUIDValue(it) } is Type.BooleanVector -> result.getObject(field.name, PgBitVector::class.java).toBooleanVector() is Type.IntVector -> result.getObject(field.name, PgVector::class.java)?.toIntVector() is Type.LongVector -> result.getObject(field.name, PgVector::class.java)?.toLongVector() From 2aa6ee38dcdf664da5482671af55987a3498aa8a Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Sun, 28 Jul 2024 16:39:18 +0200 Subject: [PATCH 04/11] Changed from File to Path --- .../database/jsonl/AbstractJsonlReader.kt | 8 ++-- .../engine/database/jsonl/JsonlConnection.kt | 11 +++--- .../database/jsonl/JsonlConnectionProvider.kt | 4 +- .../engine/database/jsonl/JsonlInitializer.kt | 35 +++++++++++------- .../engine/database/jsonl/JsonlWriter.kt | 5 ++- .../JsonlRetrievableInitializer.kt | 37 ++++++++++--------- .../retrievable/JsonlRetrievableReader.kt | 14 +++---- 7 files changed, 63 insertions(+), 51 deletions(-) diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/AbstractJsonlReader.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/AbstractJsonlReader.kt index 2ca39199..ba03d79f 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/AbstractJsonlReader.kt +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/AbstractJsonlReader.kt @@ -12,6 +12,8 @@ import org.vitrivr.engine.core.model.retrievable.Retrieved import org.vitrivr.engine.database.jsonl.model.AttributeContainerList import java.io.BufferedReader import java.io.FileReader +import java.io.InputStreamReader +import kotlin.io.path.inputStream abstract class AbstractJsonlReader( final override val field: Schema.Field<*, D>, @@ -22,7 +24,7 @@ abstract class AbstractJsonlReader( /** Prototype used to create new instances. */ protected val prototype = this.field.analyser.prototype(this.field) - private val file = connection.getFile(field) + private val path = connection.getPath(field) protected abstract fun toDescriptor(list: AttributeContainerList): D @@ -41,7 +43,7 @@ abstract class AbstractJsonlReader( override fun getAll(): Sequence { - return BufferedReader(FileReader(file)).lineSequence().mapNotNull { + return BufferedReader(InputStreamReader(path.inputStream())).lineSequence().mapNotNull { try { val list = Json.decodeFromString(it) return@mapNotNull toDescriptor(list) @@ -74,6 +76,6 @@ abstract class AbstractJsonlReader( } override fun count(): Long { - return getAll().count().toLong() + return BufferedReader(InputStreamReader(path.inputStream())).lineSequence().count().toLong() } } \ No newline at end of file diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnection.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnection.kt index d8f6a1fc..8cf018bd 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnection.kt +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnection.kt @@ -9,14 +9,15 @@ import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.database.jsonl.retrievable.JsonlRetrievableInitializer import org.vitrivr.engine.database.jsonl.retrievable.JsonlRetrievableReader import org.vitrivr.engine.database.jsonl.retrievable.JsonlRetrievableWriter -import java.io.File +import java.nio.file.Path +import kotlin.io.path.absolutePathString internal val LOGGER = logger("org.vitrivr.engine.database.jsonl.JsonlConnection") class JsonlConnection( override val schemaName: String, connectionProvider: JsonlConnectionProvider, - private val root: File + private val root: Path ) : AbstractConnection(schemaName, connectionProvider) { override fun withTransaction(action: (Unit) -> T): T { @@ -24,9 +25,9 @@ class JsonlConnection( return action(Unit) } - internal val schemaRoot = File(root, schemaName) + internal val schemaRoot = root.resolve(schemaName) - fun getFile(field: Schema.Field<*, *>) = File(schemaRoot, "${field.fieldName}.jsonl") + fun getPath(field: Schema.Field<*, *>) = schemaRoot.resolve("${field.fieldName}.jsonl") override fun getRetrievableInitializer(): RetrievableInitializer = JsonlRetrievableInitializer(this) @@ -38,7 +39,7 @@ class JsonlConnection( override fun getRetrievableReader(): RetrievableReader = reader - override fun description(): String = "JsonlConnection on '${root.absolutePath}'" + override fun description(): String = "JsonlConnection on '${root.absolutePathString()}'" override fun close() { writer.close() diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnectionProvider.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnectionProvider.kt index 89d13145..c6fbca90 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnectionProvider.kt +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnectionProvider.kt @@ -14,7 +14,7 @@ import org.vitrivr.engine.core.model.descriptor.vector.* import org.vitrivr.engine.database.jsonl.scalar.ScalarJsonlProvider import org.vitrivr.engine.database.jsonl.struct.StructJsonlProvider import org.vitrivr.engine.database.jsonl.vector.VectorJsonlProvider -import java.io.File +import java.nio.file.Path class JsonlConnectionProvider() : AbstractConnectionProvider() { @@ -26,7 +26,7 @@ class JsonlConnectionProvider() : AbstractConnectionProvider() { val rootPath = parameters["root"] ?: "." - return JsonlConnection(schemaName, this, File(rootPath)) + return JsonlConnection(schemaName, this, Path.of(rootPath)) } diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlInitializer.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlInitializer.kt index d8364c8b..dfa2e481 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlInitializer.kt +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlInitializer.kt @@ -4,36 +4,43 @@ import org.vitrivr.engine.core.database.descriptor.DescriptorInitializer import org.vitrivr.engine.core.model.descriptor.Descriptor import org.vitrivr.engine.core.model.metamodel.Schema import java.io.IOException +import kotlin.io.path.* -class JsonlInitializer(override val field: Schema.Field<*, D>, private val connection: JsonlConnection) : DescriptorInitializer { +class JsonlInitializer( + override val field: Schema.Field<*, D>, + connection: JsonlConnection +) : DescriptorInitializer { - private val file = connection.getFile(field) + private val path = connection.getPath(field) override fun initialize() { - try{ - file.parentFile.mkdirs() - file.createNewFile() + try { + path.parent.createDirectories() + path.createFile() } catch (ioe: IOException) { - LOGGER.error(ioe) { "Cannot initialize '${file.absolutePath}'" } + LOGGER.error(ioe) { "Cannot initialize '${path.absolutePathString()}'" } } catch (se: SecurityException) { - LOGGER.error(se) { "Cannot initialize '${file.absolutePath}'" } + LOGGER.error(se) { "Cannot initialize '${path.absolutePathString()}'" } } } override fun deinitialize() { - /* nop */ + try { + path.deleteExisting() + } catch (ioe: IOException) { + LOGGER.error(ioe) { "Cannot truncate '${path.absolutePathString()}'" } + } } - override fun isInitialized(): Boolean = file.exists() + override fun isInitialized(): Boolean = path.exists() override fun truncate() { - try{ - file.delete() + try { + path.deleteExisting() + path.createFile() } catch (ioe: IOException) { - LOGGER.error(ioe) { "Cannot delete '${file.absolutePath}'" } - } catch (se: SecurityException) { - LOGGER.error(se) { "Cannot delete '${file.absolutePath}'" } + LOGGER.error(ioe) { "Cannot truncate '${path.absolutePathString()}'" } } } } \ No newline at end of file diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlWriter.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlWriter.kt index 16736878..58ab3d28 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlWriter.kt +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlWriter.kt @@ -11,13 +11,14 @@ import org.vitrivr.engine.database.jsonl.JsonlConnection.Companion.DESCRIPTOR_ID import org.vitrivr.engine.database.jsonl.JsonlConnection.Companion.RETRIEVABLE_ID_COLUMN_NAME import org.vitrivr.engine.database.jsonl.model.AttributeContainer import org.vitrivr.engine.database.jsonl.model.AttributeContainerList -import java.io.FileWriter +import java.nio.file.StandardOpenOption +import kotlin.io.path.writer class JsonlWriter(override val field: Schema.Field<*, D>, override val connection: JsonlConnection) : DescriptorWriter, AutoCloseable { - private val writer = FileWriter(connection.getFile(field), true) + private val writer = connection.getPath(field).writer(Charsets.UTF_8, StandardOpenOption.APPEND) override fun add(item: D): Boolean { diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableInitializer.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableInitializer.kt index 2ff51baa..54ec74c9 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableInitializer.kt +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableInitializer.kt @@ -3,42 +3,43 @@ package org.vitrivr.engine.database.jsonl.retrievable import org.vitrivr.engine.core.database.retrievable.RetrievableInitializer import org.vitrivr.engine.database.jsonl.JsonlConnection import org.vitrivr.engine.database.jsonl.LOGGER -import java.io.File import java.io.IOException +import kotlin.io.path.* class JsonlRetrievableInitializer(private val connection: JsonlConnection) : RetrievableInitializer { - private val retrievableFile = File(connection.schemaRoot, "retrievables.jsonl") - private val connectionFile = File(connection.schemaRoot, "retrievable_connections.jsonl") + private val retrievablePath = connection.schemaRoot.resolve("retrievables.jsonl") + private val connectionPath = connection.schemaRoot.resolve("retrievable_connections.jsonl") override fun initialize() { try { - connection.schemaRoot.mkdirs() - retrievableFile.createNewFile() - connectionFile.createNewFile() + connection.schemaRoot.createDirectories() + retrievablePath.createFile() + connectionPath.createFile() } catch (ioe: IOException) { - LOGGER.error(ioe) { "Cannot initialize '${connection.schemaRoot.absolutePath}'" } - } catch (se: SecurityException) { - LOGGER.error(se) { "Cannot initialize '${connection.schemaRoot.absolutePath}'" } + LOGGER.error(ioe) { "Cannot initialize '${connection.schemaRoot.absolutePathString()}'" } } } override fun deinitialize() { - /* nop */ + try { + retrievablePath.deleteExisting() + connectionPath.deleteExisting() + } catch (ioe: IOException) { + LOGGER.error(ioe) { "Cannot delete '${connection.schemaRoot.absolutePathString()}'" } + } } - override fun isInitialized(): Boolean { - TODO("Not yet implemented") - } + override fun isInitialized(): Boolean = retrievablePath.exists() && connectionPath.exists() override fun truncate() { try { - retrievableFile.delete() - connectionFile.delete() + retrievablePath.deleteExisting() + retrievablePath.createFile() + connectionPath.deleteExisting() + connectionPath.createFile() } catch (ioe: IOException) { - LOGGER.error(ioe) { "Cannot delete '${connection.schemaRoot.absolutePath}'" } - } catch (se: SecurityException) { - LOGGER.error(se) { "Cannot delete '${connection.schemaRoot.absolutePath}'" } + LOGGER.error(ioe) { "Cannot truncate '${connection.schemaRoot.absolutePathString()}'" } } } } \ No newline at end of file diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableReader.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableReader.kt index 5da3a293..c9500365 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableReader.kt +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableReader.kt @@ -10,13 +10,13 @@ import org.vitrivr.engine.database.jsonl.LOGGER import org.vitrivr.engine.database.jsonl.model.JsonlRelationship import org.vitrivr.engine.database.jsonl.model.JsonlRetrievable import java.io.BufferedReader -import java.io.File -import java.io.FileReader +import java.io.InputStreamReader +import kotlin.io.path.inputStream class JsonlRetrievableReader(override val connection: JsonlConnection) : RetrievableReader { - private val retrievableFile = File(connection.schemaRoot, "retrievables.jsonl") - private val connectionFile = File(connection.schemaRoot, "retrievable_connections.jsonl") + private val retrievablePath = connection.schemaRoot.resolve("retrievables.jsonl") + private val connectionPath = connection.schemaRoot.resolve("retrievable_connections.jsonl") override fun get(id: RetrievableId): Retrievable? = getAll().firstOrNull { it.id == id } @@ -36,7 +36,7 @@ class JsonlRetrievableReader(override val connection: JsonlConnection) : Retriev val predIds = predicates.toSet() val objIds = objectIds.toSet() - return BufferedReader(FileReader(connectionFile)).lineSequence().mapNotNull { + return BufferedReader(InputStreamReader(connectionPath.inputStream())).lineSequence().mapNotNull { try { Json.decodeFromString(it) } catch (se: SerializationException) { @@ -54,7 +54,7 @@ class JsonlRetrievableReader(override val connection: JsonlConnection) : Retriev } override fun getAll(): Sequence { - return BufferedReader(FileReader(retrievableFile)).lineSequence().mapNotNull { + return BufferedReader(InputStreamReader(retrievablePath.inputStream())).lineSequence().mapNotNull { try { Json.decodeFromString(it).toRetrieved() } catch (se: SerializationException) { @@ -67,6 +67,6 @@ class JsonlRetrievableReader(override val connection: JsonlConnection) : Retriev } } - override fun count(): Long = getAll().count().toLong() + override fun count(): Long = BufferedReader(InputStreamReader(retrievablePath.inputStream())).lineSequence().count().toLong() } \ No newline at end of file From 0af2442924b72e50d6879fd016126737aec47102 Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Sun, 28 Jul 2024 16:51:40 +0200 Subject: [PATCH 05/11] Moved connection to its own module --- vitrivr-engine-module-jsonl/build.gradle | 68 +++++++++++++++++++ .../database/jsonl/AbstractJsonlProvider.kt | 12 +++- .../database/jsonl/AbstractJsonlReader.kt | 0 .../engine/database/jsonl/JsonlConnection.kt | 0 .../database/jsonl/JsonlConnectionProvider.kt | 0 .../engine/database/jsonl/JsonlInitializer.kt | 0 .../engine/database/jsonl/JsonlWriter.kt | 0 .../jsonl/model/AttributeContainer.kt | 0 .../jsonl/model/AttributeContainerList.kt | 0 .../database/jsonl/model/JsonlRelationship.kt | 0 .../database/jsonl/model/JsonlRetrievable.kt | 0 .../database/jsonl/model/ValueContainer.kt | 0 .../JsonlRetrievableInitializer.kt | 0 .../retrievable/JsonlRetrievableReader.kt | 0 .../retrievable/JsonlRetrievableWriter.kt | 6 +- .../jsonl/scalar/ScalarJsonlProvider.kt | 0 .../jsonl/scalar/ScalarJsonlReader.kt | 0 .../jsonl/struct/StructJsonlProvider.kt | 1 - .../jsonl/struct/StructJsonlReader.kt | 0 .../jsonl/vector/VectorJsonlProvider.kt | 0 .../jsonl/vector/VectorJsonlReader.kt | 0 ...vr.engine.core.database.ConnectionProvider | 0 22 files changed, 82 insertions(+), 5 deletions(-) create mode 100644 vitrivr-engine-module-jsonl/build.gradle rename {vitrivr-engine-index => vitrivr-engine-module-jsonl}/src/main/kotlin/org/vitrivr/engine/database/jsonl/AbstractJsonlProvider.kt (69%) rename {vitrivr-engine-index => vitrivr-engine-module-jsonl}/src/main/kotlin/org/vitrivr/engine/database/jsonl/AbstractJsonlReader.kt (100%) rename {vitrivr-engine-index => vitrivr-engine-module-jsonl}/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnection.kt (100%) rename {vitrivr-engine-index => vitrivr-engine-module-jsonl}/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnectionProvider.kt (100%) rename {vitrivr-engine-index => vitrivr-engine-module-jsonl}/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlInitializer.kt (100%) rename {vitrivr-engine-index => vitrivr-engine-module-jsonl}/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlWriter.kt (100%) rename {vitrivr-engine-index => vitrivr-engine-module-jsonl}/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/AttributeContainer.kt (100%) rename {vitrivr-engine-index => vitrivr-engine-module-jsonl}/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/AttributeContainerList.kt (100%) rename {vitrivr-engine-index => vitrivr-engine-module-jsonl}/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/JsonlRelationship.kt (100%) rename {vitrivr-engine-index => vitrivr-engine-module-jsonl}/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/JsonlRetrievable.kt (100%) rename {vitrivr-engine-index => vitrivr-engine-module-jsonl}/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/ValueContainer.kt (100%) rename {vitrivr-engine-index => vitrivr-engine-module-jsonl}/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableInitializer.kt (100%) rename {vitrivr-engine-index => vitrivr-engine-module-jsonl}/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableReader.kt (100%) rename {vitrivr-engine-index => vitrivr-engine-module-jsonl}/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableWriter.kt (87%) rename {vitrivr-engine-index => vitrivr-engine-module-jsonl}/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlProvider.kt (100%) rename {vitrivr-engine-index => vitrivr-engine-module-jsonl}/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlReader.kt (100%) rename {vitrivr-engine-index => vitrivr-engine-module-jsonl}/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlProvider.kt (90%) rename {vitrivr-engine-index => vitrivr-engine-module-jsonl}/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlReader.kt (100%) rename {vitrivr-engine-index => vitrivr-engine-module-jsonl}/src/main/kotlin/org/vitrivr/engine/database/jsonl/vector/VectorJsonlProvider.kt (100%) rename {vitrivr-engine-index => vitrivr-engine-module-jsonl}/src/main/kotlin/org/vitrivr/engine/database/jsonl/vector/VectorJsonlReader.kt (100%) rename {vitrivr-engine-index => vitrivr-engine-module-jsonl}/src/main/resources/META-INF/services/org.vitrivr.engine.core.database.ConnectionProvider (100%) diff --git a/vitrivr-engine-module-jsonl/build.gradle b/vitrivr-engine-module-jsonl/build.gradle new file mode 100644 index 00000000..ef5e253c --- /dev/null +++ b/vitrivr-engine-module-jsonl/build.gradle @@ -0,0 +1,68 @@ +plugins { + id 'org.jetbrains.kotlin.plugin.serialization' version "$version_kotlin" + id 'maven-publish' + id 'signing' +} + +repositories { + mavenCentral() +} + +dependencies { + api project(':vitrivr-engine-core') + + /** vitrivr engine Core is required for running tests. */ + testImplementation(testFixtures(project(':vitrivr-engine-core'))) +} + +/* Publication of vitrivr engine query to Maven Central. */ +publishing { + publications { + mavenJava(MavenPublication) { + groupId = 'org.vitrivr' + artifactId = 'vitrivr-engine-module-jsonl' + version = System.getenv().getOrDefault("MAVEN_PUBLICATION_VERSION", version.toString()) + from components.java + pom { + name = 'vitrivr Engine JSONL Connection Plugin' + description = 'A module that adds a file-based pseudo database using line-wise JSON.' + url = 'https://github.com/vitrivr/vitrivr-engine/' + licenses { + license { + name = 'MIT License' + } + } + developers { + developer { + id = 'ppanopticon' + name = 'Ralph Gasser' + email = 'ralph.gasser@unibas.ch' + } + developer { + id = 'lucaro' + name = 'Luca Rossetto' + email = 'rossetto@ifi.uzh.ch' + } + } + scm { + connection = 'scm:git:https://github.com/vitrivr/vitrivr-engine.git' + url = 'https://github.com/vitrivr/vitrivr-engine/' + } + } + } + } + repositories { + repositories { + maven { + def releasesRepoUrl = 'https://oss.sonatype.org/service/local/staging/deploy/maven2/' + def snapshotsRepoUrl = 'https://oss.sonatype.org/content/repositories/snapshots/' + name = "OSSRH" + url = version.endsWith('SNAPSHOT') ? snapshotsRepoUrl : releasesRepoUrl + credentials { + username = System.getenv("MAVEN_USERNAME") + password = System.getenv("MAVEN_PASSWORD") + } + } + } + } +} \ No newline at end of file diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/AbstractJsonlProvider.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/AbstractJsonlProvider.kt similarity index 69% rename from vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/AbstractJsonlProvider.kt rename to vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/AbstractJsonlProvider.kt index 2c36f349..25d5fad2 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/AbstractJsonlProvider.kt +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/AbstractJsonlProvider.kt @@ -9,7 +9,15 @@ import org.vitrivr.engine.core.model.metamodel.Schema abstract class AbstractJsonlProvider : DescriptorProvider { - override fun newInitializer(connection: Connection, field: Schema.Field<*, D>): DescriptorInitializer = JsonlInitializer(field, connection as JsonlConnection) + override fun newInitializer(connection: Connection, field: Schema.Field<*, D>): DescriptorInitializer = + JsonlInitializer( + field, + connection as JsonlConnection + ) - override fun newWriter(connection: Connection, field: Schema.Field<*, D>): DescriptorWriter = JsonlWriter(field, connection as JsonlConnection) + override fun newWriter(connection: Connection, field: Schema.Field<*, D>): DescriptorWriter = + JsonlWriter( + field, + connection as JsonlConnection + ) } \ No newline at end of file diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/AbstractJsonlReader.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/AbstractJsonlReader.kt similarity index 100% rename from vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/AbstractJsonlReader.kt rename to vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/AbstractJsonlReader.kt diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnection.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnection.kt similarity index 100% rename from vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnection.kt rename to vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnection.kt diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnectionProvider.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnectionProvider.kt similarity index 100% rename from vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnectionProvider.kt rename to vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnectionProvider.kt diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlInitializer.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlInitializer.kt similarity index 100% rename from vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlInitializer.kt rename to vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlInitializer.kt diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlWriter.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlWriter.kt similarity index 100% rename from vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlWriter.kt rename to vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlWriter.kt diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/AttributeContainer.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/AttributeContainer.kt similarity index 100% rename from vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/AttributeContainer.kt rename to vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/AttributeContainer.kt diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/AttributeContainerList.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/AttributeContainerList.kt similarity index 100% rename from vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/AttributeContainerList.kt rename to vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/AttributeContainerList.kt diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/JsonlRelationship.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/JsonlRelationship.kt similarity index 100% rename from vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/JsonlRelationship.kt rename to vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/JsonlRelationship.kt diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/JsonlRetrievable.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/JsonlRetrievable.kt similarity index 100% rename from vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/JsonlRetrievable.kt rename to vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/JsonlRetrievable.kt diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/ValueContainer.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/ValueContainer.kt similarity index 100% rename from vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/ValueContainer.kt rename to vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/ValueContainer.kt diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableInitializer.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableInitializer.kt similarity index 100% rename from vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableInitializer.kt rename to vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableInitializer.kt diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableReader.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableReader.kt similarity index 100% rename from vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableReader.kt rename to vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableReader.kt diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableWriter.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableWriter.kt similarity index 87% rename from vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableWriter.kt rename to vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableWriter.kt index 357e9cd2..fba04529 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableWriter.kt +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableWriter.kt @@ -11,11 +11,13 @@ import org.vitrivr.engine.database.jsonl.model.JsonlRelationship import org.vitrivr.engine.database.jsonl.model.JsonlRetrievable import java.io.File import java.io.FileWriter +import java.nio.file.StandardOpenOption +import kotlin.io.path.writer class JsonlRetrievableWriter(override val connection: JsonlConnection) : RetrievableWriter, AutoCloseable { - private val retrievableWriter = FileWriter(File(connection.schemaRoot, "retrievables.jsonl"), true) - private val connectionWriter = FileWriter(File(connection.schemaRoot, "retrievable_connections.jsonl"), true) + private val retrievableWriter = connection.schemaRoot.resolve("retrievables.jsonl").writer(Charsets.UTF_8, StandardOpenOption.APPEND) + private val connectionWriter = connection.schemaRoot.resolve("retrievable_connections.jsonl").writer(Charsets.UTF_8, StandardOpenOption.APPEND) @Synchronized override fun connect(relationship: Relationship): Boolean { diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlProvider.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlProvider.kt similarity index 100% rename from vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlProvider.kt rename to vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlProvider.kt diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlReader.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlReader.kt similarity index 100% rename from vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlReader.kt rename to vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlReader.kt diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlProvider.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlProvider.kt similarity index 90% rename from vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlProvider.kt rename to vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlProvider.kt index 9c4507c6..0d9a9b50 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlProvider.kt +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlProvider.kt @@ -3,7 +3,6 @@ package org.vitrivr.engine.database.jsonl.struct import org.vitrivr.engine.core.database.Connection import org.vitrivr.engine.core.database.descriptor.DescriptorReader import org.vitrivr.engine.core.model.descriptor.struct.StructDescriptor -import org.vitrivr.engine.core.model.descriptor.vector.VectorDescriptor import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.database.jsonl.AbstractJsonlProvider import org.vitrivr.engine.database.jsonl.JsonlConnection diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlReader.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlReader.kt similarity index 100% rename from vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlReader.kt rename to vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlReader.kt diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/vector/VectorJsonlProvider.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/vector/VectorJsonlProvider.kt similarity index 100% rename from vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/vector/VectorJsonlProvider.kt rename to vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/vector/VectorJsonlProvider.kt diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/vector/VectorJsonlReader.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/vector/VectorJsonlReader.kt similarity index 100% rename from vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/database/jsonl/vector/VectorJsonlReader.kt rename to vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/vector/VectorJsonlReader.kt diff --git a/vitrivr-engine-index/src/main/resources/META-INF/services/org.vitrivr.engine.core.database.ConnectionProvider b/vitrivr-engine-module-jsonl/src/main/resources/META-INF/services/org.vitrivr.engine.core.database.ConnectionProvider similarity index 100% rename from vitrivr-engine-index/src/main/resources/META-INF/services/org.vitrivr.engine.core.database.ConnectionProvider rename to vitrivr-engine-module-jsonl/src/main/resources/META-INF/services/org.vitrivr.engine.core.database.ConnectionProvider From 8d03253fd517d53363ed7966676d1ef5fb5e0abb Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Mon, 29 Jul 2024 20:22:44 +0200 Subject: [PATCH 06/11] Added test suite --- .../engine/database/jsonl/JsonlInitializer.kt | 4 +- .../JsonlRetrievableInitializer.kt | 8 ++-- .../descriptor/DescriptorInitializerTest.kt | 6 +++ .../FileMetadataDescriptorReaderTest.kt | 6 +++ .../vector/FloatVectorDescriptorReaderTest.kt | 6 +++ .../retrievable/RetrievableInitializerTest.kt | 8 ++++ .../retrievable/RetrievableWriterTest.kt | 7 ++++ .../src/test/resources/test-schema-jsonl.json | 42 +++++++++++++++++++ 8 files changed, 81 insertions(+), 6 deletions(-) create mode 100644 vitrivr-engine-module-jsonl/src/test/kotlin/org/vitrivr/engine/database/jsonl/descriptor/DescriptorInitializerTest.kt create mode 100644 vitrivr-engine-module-jsonl/src/test/kotlin/org/vitrivr/engine/database/jsonl/descriptor/struct/FileMetadataDescriptorReaderTest.kt create mode 100644 vitrivr-engine-module-jsonl/src/test/kotlin/org/vitrivr/engine/database/jsonl/descriptor/vector/FloatVectorDescriptorReaderTest.kt create mode 100644 vitrivr-engine-module-jsonl/src/test/kotlin/org/vitrivr/engine/database/jsonl/retrievable/RetrievableInitializerTest.kt create mode 100644 vitrivr-engine-module-jsonl/src/test/kotlin/org/vitrivr/engine/database/jsonl/retrievable/RetrievableWriterTest.kt create mode 100644 vitrivr-engine-module-jsonl/src/test/resources/test-schema-jsonl.json diff --git a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlInitializer.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlInitializer.kt index dfa2e481..68b7a113 100644 --- a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlInitializer.kt +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlInitializer.kt @@ -27,9 +27,9 @@ class JsonlInitializer( override fun deinitialize() { try { - path.deleteExisting() + path.deleteIfExists() } catch (ioe: IOException) { - LOGGER.error(ioe) { "Cannot truncate '${path.absolutePathString()}'" } + LOGGER.error(ioe) { "Cannot deinitialize '${path.absolutePathString()}'" } } } diff --git a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableInitializer.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableInitializer.kt index 54ec74c9..97c46252 100644 --- a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableInitializer.kt +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableInitializer.kt @@ -23,8 +23,8 @@ class JsonlRetrievableInitializer(private val connection: JsonlConnection) : Ret override fun deinitialize() { try { - retrievablePath.deleteExisting() - connectionPath.deleteExisting() + retrievablePath.deleteIfExists() + connectionPath.deleteIfExists() } catch (ioe: IOException) { LOGGER.error(ioe) { "Cannot delete '${connection.schemaRoot.absolutePathString()}'" } } @@ -34,9 +34,9 @@ class JsonlRetrievableInitializer(private val connection: JsonlConnection) : Ret override fun truncate() { try { - retrievablePath.deleteExisting() + retrievablePath.deleteIfExists() retrievablePath.createFile() - connectionPath.deleteExisting() + connectionPath.deleteIfExists() connectionPath.createFile() } catch (ioe: IOException) { LOGGER.error(ioe) { "Cannot truncate '${connection.schemaRoot.absolutePathString()}'" } diff --git a/vitrivr-engine-module-jsonl/src/test/kotlin/org/vitrivr/engine/database/jsonl/descriptor/DescriptorInitializerTest.kt b/vitrivr-engine-module-jsonl/src/test/kotlin/org/vitrivr/engine/database/jsonl/descriptor/DescriptorInitializerTest.kt new file mode 100644 index 00000000..37222081 --- /dev/null +++ b/vitrivr-engine-module-jsonl/src/test/kotlin/org/vitrivr/engine/database/jsonl/descriptor/DescriptorInitializerTest.kt @@ -0,0 +1,6 @@ +package org.vitrivr.engine.database.jsonl.descriptor + +import org.vitrivr.engine.core.database.descriptor.AbstractDescriptorInitializerTest + + +class DescriptorInitializerTest : AbstractDescriptorInitializerTest("test-schema-jsonl.json") \ No newline at end of file diff --git a/vitrivr-engine-module-jsonl/src/test/kotlin/org/vitrivr/engine/database/jsonl/descriptor/struct/FileMetadataDescriptorReaderTest.kt b/vitrivr-engine-module-jsonl/src/test/kotlin/org/vitrivr/engine/database/jsonl/descriptor/struct/FileMetadataDescriptorReaderTest.kt new file mode 100644 index 00000000..87d1c796 --- /dev/null +++ b/vitrivr-engine-module-jsonl/src/test/kotlin/org/vitrivr/engine/database/jsonl/descriptor/struct/FileMetadataDescriptorReaderTest.kt @@ -0,0 +1,6 @@ +package org.vitrivr.engine.database.jsonl.descriptor.struct + +import org.vitrivr.engine.core.database.descriptor.struct.AbstractFileMetadataDescriptorReaderTest + + +class FileMetadataDescriptorReaderTest : AbstractFileMetadataDescriptorReaderTest("test-schema-jsonl.json") \ No newline at end of file diff --git a/vitrivr-engine-module-jsonl/src/test/kotlin/org/vitrivr/engine/database/jsonl/descriptor/vector/FloatVectorDescriptorReaderTest.kt b/vitrivr-engine-module-jsonl/src/test/kotlin/org/vitrivr/engine/database/jsonl/descriptor/vector/FloatVectorDescriptorReaderTest.kt new file mode 100644 index 00000000..768645f0 --- /dev/null +++ b/vitrivr-engine-module-jsonl/src/test/kotlin/org/vitrivr/engine/database/jsonl/descriptor/vector/FloatVectorDescriptorReaderTest.kt @@ -0,0 +1,6 @@ +package org.vitrivr.engine.database.jsonl.descriptor.vector + +import org.vitrivr.engine.core.database.descriptor.vector.AbstractFloatVectorDescriptorReaderTest + + +class FloatVectorDescriptorReaderTest : AbstractFloatVectorDescriptorReaderTest("test-schema-jsonl.json") \ No newline at end of file diff --git a/vitrivr-engine-module-jsonl/src/test/kotlin/org/vitrivr/engine/database/jsonl/retrievable/RetrievableInitializerTest.kt b/vitrivr-engine-module-jsonl/src/test/kotlin/org/vitrivr/engine/database/jsonl/retrievable/RetrievableInitializerTest.kt new file mode 100644 index 00000000..872b96dd --- /dev/null +++ b/vitrivr-engine-module-jsonl/src/test/kotlin/org/vitrivr/engine/database/jsonl/retrievable/RetrievableInitializerTest.kt @@ -0,0 +1,8 @@ +package org.vitrivr.engine.database.jsonl.retrievable + +import org.vitrivr.engine.core.database.retrievable.AbstractRetrievableInitializerTest + + +class RetrievableInitializerTest : AbstractRetrievableInitializerTest("test-schema-jsonl.json") { + +} \ No newline at end of file diff --git a/vitrivr-engine-module-jsonl/src/test/kotlin/org/vitrivr/engine/database/jsonl/retrievable/RetrievableWriterTest.kt b/vitrivr-engine-module-jsonl/src/test/kotlin/org/vitrivr/engine/database/jsonl/retrievable/RetrievableWriterTest.kt new file mode 100644 index 00000000..acaac7a1 --- /dev/null +++ b/vitrivr-engine-module-jsonl/src/test/kotlin/org/vitrivr/engine/database/jsonl/retrievable/RetrievableWriterTest.kt @@ -0,0 +1,7 @@ +package org.vitrivr.engine.database.jsonl.retrievable + +import org.vitrivr.engine.core.database.retrievable.AbstractRetrievableWriterTest + +class RetrievableWriterTest : AbstractRetrievableWriterTest("test-schema-jsonl.json") { + +} \ No newline at end of file diff --git a/vitrivr-engine-module-jsonl/src/test/resources/test-schema-jsonl.json b/vitrivr-engine-module-jsonl/src/test/resources/test-schema-jsonl.json new file mode 100644 index 00000000..bd000f29 --- /dev/null +++ b/vitrivr-engine-module-jsonl/src/test/resources/test-schema-jsonl.json @@ -0,0 +1,42 @@ +{ + "name": "vitrivr-test", + "connection": { + "database": "JsonlConnectionProvider", + "parameters": { + "root": "jsonl-test" + } + }, + "fields": [ + { + "name": "averagecolor", + "factory": "AverageColor" + }, + { + "name": "file", + "factory": "FileSourceMetadata", + "indexes": [ + { + "attributes": [ + "path" + ], + "type": "FULLTEXT" + }, + { + "attributes": [ + "size" + ], + "type": "SCALAR" + } + ] + }, + { + "name": "time", + "factory": "TemporalMetadata" + }, + { + "name": "video", + "factory": "VideoSourceMetadata" + } + ], + "exporters": [] +} From 0d0998a52d429a4483030863afa29f325cdfe273 Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Tue, 30 Jul 2024 17:51:15 +0200 Subject: [PATCH 07/11] Added comparison logic to ComparisonOperator --- .../model/query/basics/ComparisonOperator.kt | 129 ++++++++++++++++-- .../database/jsonl/AbstractJsonlReader.kt | 3 - .../jsonl/scalar/ScalarJsonlReader.kt | 21 ++- .../jsonl/struct/StructJsonlReader.kt | 24 +++- .../jsonl/vector/VectorJsonlReader.kt | 13 ++ 5 files changed, 171 insertions(+), 19 deletions(-) diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/query/basics/ComparisonOperator.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/query/basics/ComparisonOperator.kt index ae3a0d37..9016a4af 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/query/basics/ComparisonOperator.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/query/basics/ComparisonOperator.kt @@ -1,32 +1,134 @@ package org.vitrivr.engine.core.model.query.basics import org.vitrivr.engine.core.model.query.bool.SimpleBooleanQuery +import org.vitrivr.engine.core.model.types.Value +import java.util.* /** * Enumeration of comparison operators used by the [SimpleBooleanQuery]. * * @author Ralph Gasser * @author Loris Sauter - * @version 1.1.0 + * @author Luca Rossetto + * @version 1.2.0 */ enum class ComparisonOperator(val value: String) { - EQ("=="), - NEQ("!="), - LE("<"), - GR(">"), - LEQ("<="), - GEQ(">="), - LIKE("~="); - - companion object{ + EQ("==") { + override fun compare(v1: Value, v2: Value<*>): Boolean = v1.value == v2.value + }, + NEQ("!=") { + override fun compare(v1: Value, v2: Value<*>): Boolean = v1.value != v2.value + }, + LE("<") { + override fun compare(v1: Value, v2: Value<*>): Boolean = + when (v1) { + is Value.String -> v1.value < (v2.value as String) + is Value.Boolean -> v1.value < (v2.value as Boolean) + is Value.Byte -> v1.value < (v2.value as Byte) + is Value.DateTime -> v1.value < (v2.value as Date) + is Value.Double -> v1.value < (v2.value as Double) + is Value.Float -> v1.value < (v2.value as Float) + is Value.Int -> v1.value < (v2.value as Int) + is Value.Long -> v1.value < (v2.value as Long) + is Value.Short -> v1.value < (v2.value as Short) + is Value.Text -> v1.value < (v2.value as String) + is Value.UUIDValue -> v1.value < (v2.value as UUID) + is Value.BooleanVector, + is Value.DoubleVector, + is Value.FloatVector, + is Value.IntVector, + is Value.LongVector -> false + } + + }, + GR(">") { + override fun compare(v1: Value, v2: Value<*>): Boolean = + when (v1) { + is Value.String -> v1.value > (v2.value as String) + is Value.Boolean -> v1.value > (v2.value as Boolean) + is Value.Byte -> v1.value > (v2.value as Byte) + is Value.DateTime -> v1.value > (v2.value as Date) + is Value.Double -> v1.value > (v2.value as Double) + is Value.Float -> v1.value > (v2.value as Float) + is Value.Int -> v1.value > (v2.value as Int) + is Value.Long -> v1.value > (v2.value as Long) + is Value.Short -> v1.value > (v2.value as Short) + is Value.Text -> v1.value > (v2.value as String) + is Value.UUIDValue -> v1.value > (v2.value as UUID) + is Value.BooleanVector, + is Value.DoubleVector, + is Value.FloatVector, + is Value.IntVector, + is Value.LongVector -> false + } + + }, + LEQ("<=") { + override fun compare(v1: Value, v2: Value<*>): Boolean = + when (v1) { + is Value.String -> v1.value <= (v2.value as String) + is Value.Boolean -> v1.value <= (v2.value as Boolean) + is Value.Byte -> v1.value <= (v2.value as Byte) + is Value.DateTime -> v1.value <= (v2.value as Date) + is Value.Double -> v1.value <= (v2.value as Double) + is Value.Float -> v1.value <= (v2.value as Float) + is Value.Int -> v1.value <= (v2.value as Int) + is Value.Long -> v1.value <= (v2.value as Long) + is Value.Short -> v1.value <= (v2.value as Short) + is Value.Text -> v1.value <= (v2.value as String) + is Value.UUIDValue -> v1.value <= (v2.value as UUID) + is Value.BooleanVector, + is Value.DoubleVector, + is Value.FloatVector, + is Value.IntVector, + is Value.LongVector -> false + } + + }, + GEQ(">=") { + override fun compare(v1: Value, v2: Value<*>): Boolean = + when (v1) { + is Value.String -> v1.value >= (v2.value as String) + is Value.Boolean -> v1.value >= (v2.value as Boolean) + is Value.Byte -> v1.value >= (v2.value as Byte) + is Value.DateTime -> v1.value >= (v2.value as Date) + is Value.Double -> v1.value >= (v2.value as Double) + is Value.Float -> v1.value >= (v2.value as Float) + is Value.Int -> v1.value >= (v2.value as Int) + is Value.Long -> v1.value >= (v2.value as Long) + is Value.Short -> v1.value >= (v2.value as Short) + is Value.Text -> v1.value >= (v2.value as String) + is Value.UUIDValue -> v1.value <= (v2.value as UUID) + is Value.BooleanVector, + is Value.DoubleVector, + is Value.FloatVector, + is Value.IntVector, + is Value.LongVector -> false + } + + }, + LIKE("~=") { + override fun compare(v1: Value, v2: Value<*>): Boolean = + when (v1) { + is Value.String, + is Value.Text -> { + (v1.value as String).replace("*", "\\*").replace("%", "*").toRegex().matches(v2.value as String) + } + + else -> false + } + + }; + + companion object { /** * Resolves a [ComparisonOperator] from the given [String]. * * @param str The [String] which should be one of the [ComparisonOperator] * @throws IllegalArgumentException In case the given string is not one of the defined ones. */ - fun fromString(str: String):ComparisonOperator{ - return when(str.trim()){ + fun fromString(str: String): ComparisonOperator { + return when (str.trim()) { EQ.value -> EQ NEQ.value -> NEQ LE.value -> LE @@ -38,4 +140,7 @@ enum class ComparisonOperator(val value: String) { } } } + + abstract fun compare(v1: Value, v2: Value<*>): Boolean + } diff --git a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/AbstractJsonlReader.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/AbstractJsonlReader.kt index ba03d79f..44151b0e 100644 --- a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/AbstractJsonlReader.kt +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/AbstractJsonlReader.kt @@ -67,9 +67,6 @@ abstract class AbstractJsonlReader( return getAll().filter { ids.contains(it.retrievableId) } } - override fun query(query: Query): Sequence { - TODO("Not yet implemented") - } override fun queryAndJoin(query: Query): Sequence { TODO("Not yet implemented") diff --git a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlReader.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlReader.kt index caf1a50c..67424bbe 100644 --- a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlReader.kt +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlReader.kt @@ -2,6 +2,10 @@ package org.vitrivr.engine.database.jsonl.scalar import org.vitrivr.engine.core.model.descriptor.scalar.* import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.query.Query +import org.vitrivr.engine.core.model.query.basics.ComparisonOperator +import org.vitrivr.engine.core.model.query.bool.SimpleBooleanQuery +import org.vitrivr.engine.core.model.query.fulltext.SimpleFulltextQuery import org.vitrivr.engine.core.model.types.Value import org.vitrivr.engine.database.jsonl.AbstractJsonlReader import org.vitrivr.engine.database.jsonl.model.AttributeContainerList @@ -21,7 +25,7 @@ class ScalarJsonlReader( val descriptorId = (map[RETRIEVABLE_ID_COLUMN_NAME]?.value!!.toValue() as Value.UUIDValue).value val value = map["value"]?.value!!.toValue() - return when(prototype) { + return when (prototype) { is BooleanDescriptor -> BooleanDescriptor(retrievableId, descriptorId, value as Value.Boolean) is DoubleDescriptor -> DoubleDescriptor(retrievableId, descriptorId, value as Value.Double) is FloatDescriptor -> FloatDescriptor(retrievableId, descriptorId, value as Value.Float) @@ -33,7 +37,20 @@ class ScalarJsonlReader( } } + } + + override fun query(query: Query): Sequence> = when (query) { + is SimpleFulltextQuery -> this.queryFulltext(query) + is SimpleBooleanQuery<*> -> this.queryBoolean(query) + else -> throw UnsupportedOperationException("The provided query type ${query::class.simpleName} is not supported by this reader.") + } + private fun queryFulltext(fulltextQuery: SimpleFulltextQuery): Sequence> { + TODO() } -} \ No newline at end of file + private fun queryBoolean(query: SimpleBooleanQuery<*>): Sequence> = + getAll().filter { descriptor -> + query.comparison.compare(query.value, descriptor.value) + } +} diff --git a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlReader.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlReader.kt index e7a6785d..a7a6b177 100644 --- a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlReader.kt +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlReader.kt @@ -3,6 +3,9 @@ package org.vitrivr.engine.database.jsonl.struct import org.vitrivr.engine.core.model.descriptor.AttributeName import org.vitrivr.engine.core.model.descriptor.struct.StructDescriptor import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.query.Query +import org.vitrivr.engine.core.model.query.bool.SimpleBooleanQuery +import org.vitrivr.engine.core.model.query.fulltext.SimpleFulltextQuery import org.vitrivr.engine.core.model.types.Value import org.vitrivr.engine.database.jsonl.AbstractJsonlReader import org.vitrivr.engine.database.jsonl.JsonlConnection @@ -11,7 +14,7 @@ import org.vitrivr.engine.database.jsonl.JsonlConnection.Companion.RETRIEVABLE_I import org.vitrivr.engine.database.jsonl.model.AttributeContainerList import kotlin.reflect.full.primaryConstructor -class StructJsonlReader ( +class StructJsonlReader( field: Schema.Field<*, StructDescriptor>, connection: JsonlConnection ) : AbstractJsonlReader(field, connection) { @@ -19,7 +22,8 @@ class StructJsonlReader ( override fun toDescriptor(list: AttributeContainerList): StructDescriptor { val map = list.list.associateBy { it.attribute.name } - val constructor = this.field.analyser.descriptorClass.primaryConstructor ?: throw IllegalStateException("Provided type ${this.field.analyser.descriptorClass} does not have a primary constructor.") + val constructor = this.field.analyser.descriptorClass.primaryConstructor + ?: throw IllegalStateException("Provided type ${this.field.analyser.descriptorClass} does not have a primary constructor.") val valueMap = mutableMapOf>() val retrievableId = (map[DESCRIPTOR_ID_COLUMN_NAME]?.value!!.toValue() as Value.UUIDValue).value @@ -37,4 +41,20 @@ class StructJsonlReader ( return constructor.call(*parameters.toTypedArray()) } + + override fun query(query: Query): Sequence = when (query) { + is SimpleFulltextQuery -> this.queryFulltext(query) + is SimpleBooleanQuery<*> -> this.queryBoolean(query) + else -> throw UnsupportedOperationException("The provided query type ${query::class.simpleName} is not supported by this reader.") + } + + private fun queryFulltext(fulltextQuery: SimpleFulltextQuery): Sequence { + TODO() + } + + private fun queryBoolean(query: SimpleBooleanQuery<*>): Sequence = getAll().filter { descriptor -> + query.comparison.compare(query.value, descriptor.values()[query.attributeName!!]!!) + } + + } \ No newline at end of file diff --git a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/vector/VectorJsonlReader.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/vector/VectorJsonlReader.kt index cb1420e0..f45ddc04 100644 --- a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/vector/VectorJsonlReader.kt +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/vector/VectorJsonlReader.kt @@ -2,6 +2,8 @@ package org.vitrivr.engine.database.jsonl.vector import org.vitrivr.engine.core.model.descriptor.vector.* import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.query.Query +import org.vitrivr.engine.core.model.query.proximity.ProximityQuery import org.vitrivr.engine.core.model.types.Value import org.vitrivr.engine.database.jsonl.AbstractJsonlReader import org.vitrivr.engine.database.jsonl.model.AttributeContainerList @@ -53,4 +55,15 @@ class VectorJsonlReader( ) } } + + override fun query(query: Query): Sequence> = when (query) { + is ProximityQuery<*> -> queryProximity(query) + else -> throw UnsupportedOperationException("Query of typ ${query::class} is not supported by this reader.") + } + + private fun queryProximity(query: ProximityQuery<*>): Sequence> { + TODO() + } + + } \ No newline at end of file From de1e4f8e173c081bde84ba51f8d0f4b3d5d310a7 Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Tue, 30 Jul 2024 18:12:50 +0200 Subject: [PATCH 08/11] Fixed some serialization issues --- .../kotlin/org/vitrivr/engine/core/model/types/Type.kt | 1 + .../org/vitrivr/engine/database/jsonl/JsonlWriter.kt | 9 ++++++++- .../engine/database/jsonl/model/ValueContainer.kt | 2 +- .../engine/database/jsonl/struct/StructJsonlReader.kt | 1 + 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/types/Type.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/types/Type.kt index ff4d7b39..c38410f9 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/types/Type.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/types/Type.kt @@ -10,6 +10,7 @@ import java.util.* * @author Loris Sauter * @version 2.0.0 */ +@Serializable sealed interface Type { companion object { diff --git a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlWriter.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlWriter.kt index 58ab3d28..44b5ca44 100644 --- a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlWriter.kt +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlWriter.kt @@ -3,9 +3,11 @@ package org.vitrivr.engine.database.jsonl import kotlinx.serialization.encodeToString import kotlinx.serialization.json.Json import org.vitrivr.engine.core.database.descriptor.DescriptorWriter +import org.vitrivr.engine.core.model.descriptor.Attribute import org.vitrivr.engine.core.model.descriptor.AttributeName import org.vitrivr.engine.core.model.descriptor.Descriptor import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.types.Type import org.vitrivr.engine.core.model.types.Value import org.vitrivr.engine.database.jsonl.JsonlConnection.Companion.DESCRIPTOR_ID_COLUMN_NAME import org.vitrivr.engine.database.jsonl.JsonlConnection.Companion.RETRIEVABLE_ID_COLUMN_NAME @@ -31,9 +33,14 @@ class JsonlWriter(override val field: Schema.Field<*, D>, overri ) valueMap.putAll(item.values()) + val attributes = mutableListOf( + Attribute(DESCRIPTOR_ID_COLUMN_NAME, Type.UUID, false), + Attribute(RETRIEVABLE_ID_COLUMN_NAME, Type.UUID, false) + ) + attributes.addAll(item.layout()) val list = AttributeContainerList( - item.layout().map { attribute -> + attributes.map { attribute -> AttributeContainer( attribute, valueMap[attribute.name] diff --git a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/ValueContainer.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/ValueContainer.kt index f66b9377..6e0c25d7 100644 --- a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/ValueContainer.kt +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/ValueContainer.kt @@ -8,7 +8,7 @@ import org.vitrivr.engine.core.model.types.Value import java.util.* @Serializable -sealed class ValueContainer(val type: Type) { +sealed class ValueContainer(val innerType: Type) { //TODO explicitly use innerType for serialization companion object { fun fromValue(value: Value<*>): ValueContainer = when (value) { diff --git a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlReader.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlReader.kt index a7a6b177..bbbf1aa4 100644 --- a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlReader.kt +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlReader.kt @@ -31,6 +31,7 @@ class StructJsonlReader( val parameters: MutableList = mutableListOf( descriptorId, retrievableId, + prototype.layout(), valueMap ) From 539aa59053c0156c94c39259d21f3dbceedd5032 Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Tue, 30 Jul 2024 18:27:05 +0200 Subject: [PATCH 09/11] Fixed struct descriptor deserialization --- .../engine/core/model/query/basics/ComparisonOperator.kt | 2 +- .../engine/database/jsonl/scalar/ScalarJsonlReader.kt | 2 +- .../engine/database/jsonl/struct/StructJsonlReader.kt | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/query/basics/ComparisonOperator.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/query/basics/ComparisonOperator.kt index 9016a4af..362236ae 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/query/basics/ComparisonOperator.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/query/basics/ComparisonOperator.kt @@ -112,7 +112,7 @@ enum class ComparisonOperator(val value: String) { when (v1) { is Value.String, is Value.Text -> { - (v1.value as String).replace("*", "\\*").replace("%", "*").toRegex().matches(v2.value as String) + (v1.value as String).replace("\\", "\\\\").replace("*", "\\*").replace("%", "*").toRegex().matches(v2.value as String) } else -> false diff --git a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlReader.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlReader.kt index 67424bbe..272096b1 100644 --- a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlReader.kt +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlReader.kt @@ -51,6 +51,6 @@ class ScalarJsonlReader( private fun queryBoolean(query: SimpleBooleanQuery<*>): Sequence> = getAll().filter { descriptor -> - query.comparison.compare(query.value, descriptor.value) + query.comparison.compare(descriptor.value, query.value) } } diff --git a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlReader.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlReader.kt index bbbf1aa4..1876e5b8 100644 --- a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlReader.kt +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlReader.kt @@ -31,8 +31,8 @@ class StructJsonlReader( val parameters: MutableList = mutableListOf( descriptorId, retrievableId, - prototype.layout(), - valueMap + valueMap, + this.field ) prototype.layout().forEach { attribute -> @@ -54,7 +54,7 @@ class StructJsonlReader( } private fun queryBoolean(query: SimpleBooleanQuery<*>): Sequence = getAll().filter { descriptor -> - query.comparison.compare(query.value, descriptor.values()[query.attributeName!!]!!) + query.comparison.compare(descriptor.values()[query.attributeName!!]!!, query.value) } From 8d0f859e97669f9521c79e015349a02f1fff7dff Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Wed, 31 Jul 2024 09:42:47 +0200 Subject: [PATCH 10/11] Added support for fulltext and started work on knn queries --- settings.gradle | 1 + .../model/query/basics/ComparisonOperator.kt | 3 +- .../core/util/knn/FixedSizePriorityQueue.kt | 38 ++++++++++++++++++ .../database/jsonl/model/ValueContainer.kt | 35 ++++++++--------- .../jsonl/scalar/ScalarJsonlReader.kt | 9 ++++- .../jsonl/struct/StructJsonlReader.kt | 9 ++++- .../jsonl/vector/VectorJsonlReader.kt | 39 ++++++++++++++++++- 7 files changed, 112 insertions(+), 22 deletions(-) create mode 100644 vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/util/knn/FixedSizePriorityQueue.kt diff --git a/settings.gradle b/settings.gradle index 0ad7ee17..62164786 100644 --- a/settings.gradle +++ b/settings.gradle @@ -8,6 +8,7 @@ include 'vitrivr-engine-index' include 'vitrivr-engine-query' include 'vitrivr-engine-server' include 'vitrivr-engine-module-cottontaildb' +include 'vitrivr-engine-module-jsonl' include 'vitrivr-engine-module-pgvector' include 'vitrivr-engine-module-features' include 'vitrivr-engine-module-m3d' diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/query/basics/ComparisonOperator.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/query/basics/ComparisonOperator.kt index 362236ae..498062bb 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/query/basics/ComparisonOperator.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/query/basics/ComparisonOperator.kt @@ -112,7 +112,8 @@ enum class ComparisonOperator(val value: String) { when (v1) { is Value.String, is Value.Text -> { - (v1.value as String).replace("\\", "\\\\").replace("*", "\\*").replace("%", "*").toRegex().matches(v2.value as String) + (v1.value as String).replace("\\", "\\\\").replace("[", "\\[").replace("]", "\\]") + .replace("*", "\\*").replace("%", "*").toRegex().matches(v2.value as String) } else -> false diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/util/knn/FixedSizePriorityQueue.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/util/knn/FixedSizePriorityQueue.kt new file mode 100644 index 00000000..ccbd0896 --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/util/knn/FixedSizePriorityQueue.kt @@ -0,0 +1,38 @@ +package org.vitrivr.engine.core.util.knn + +import java.util.* + +/** + * Ordered List of fixed size, used for KNN operations + */ + +class FixedSizePriorityQueue(private val maxSize: Int, comparator: Comparator) : TreeSet(comparator) { + + init { + require(maxSize > 0) { "Maximum size must be greater than zero." } + } + + private val elementsLeft: Int + get() = this.maxSize - this.size + + override fun add(element: T): Boolean { + if (elementsLeft > 0) { + // queue isn't full => add element and decrement elementsLeft + val added = super.add(element) + return added + } else { + // there is already 1 or more elements => compare to the least + val compared = super.comparator().compare(this.last(), element) + if (compared > 0) { + // new element is larger than the least in queue => pull the least and add new one to queue + pollLast() + super.add(element) + return true + } else { + // new element is less than the least in queue => return false + return false + } + } + } + +} \ No newline at end of file diff --git a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/ValueContainer.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/ValueContainer.kt index 6e0c25d7..aa135690 100644 --- a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/ValueContainer.kt +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/ValueContainer.kt @@ -3,12 +3,11 @@ package org.vitrivr.engine.database.jsonl.model import kotlinx.serialization.Serializable import org.vitrivr.engine.core.model.serializer.DateSerializer import org.vitrivr.engine.core.model.serializer.UUIDSerializer -import org.vitrivr.engine.core.model.types.Type import org.vitrivr.engine.core.model.types.Value import java.util.* @Serializable -sealed class ValueContainer(val innerType: Type) { //TODO explicitly use innerType for serialization +sealed class ValueContainer { companion object { fun fromValue(value: Value<*>): ValueContainer = when (value) { @@ -36,82 +35,82 @@ sealed class ValueContainer(val innerType: Type) { //TODO explicitly use innerTy } @Serializable -class BooleanValueContainer(private val value: Boolean) : ValueContainer(Type.Boolean) { +class BooleanValueContainer(private val value: Boolean) : ValueContainer() { override fun toValue(): Value = Value.Boolean(value) } @Serializable -class ByteValueContainer(private val value: Byte) : ValueContainer(Type.Byte) { +class ByteValueContainer(private val value: Byte) : ValueContainer() { override fun toValue(): Value = Value.Byte(value) } @Serializable class DateTimeValueContainer(@Serializable(DateSerializer::class) private val value: Date) : - ValueContainer(Type.Datetime) { + ValueContainer() { override fun toValue(): Value = Value.DateTime(value) } @Serializable -class DoubleValueContainer(private val value: Double) : ValueContainer(Type.Double) { +class DoubleValueContainer(private val value: Double) : ValueContainer() { override fun toValue(): Value = Value.Double(value) } @Serializable -class FloatValueContainer(private val value: Float) : ValueContainer(Type.Float) { +class FloatValueContainer(private val value: Float) : ValueContainer() { override fun toValue(): Value = Value.Float(value) } @Serializable -class IntValueContainer(private val value: Int) : ValueContainer(Type.Int) { +class IntValueContainer(private val value: Int) : ValueContainer() { override fun toValue(): Value = Value.Int(value) } @Serializable -class LongValueContainer(private val value: Long) : ValueContainer(Type.Long) { +class LongValueContainer(private val value: Long) : ValueContainer() { override fun toValue(): Value = Value.Long(value) } @Serializable -class ShortValueContainer(private val value: Short) : ValueContainer(Type.Short) { +class ShortValueContainer(private val value: Short) : ValueContainer() { override fun toValue(): Value = Value.Short(value) } @Serializable -class StringValueContainer(private val value: String) : ValueContainer(Type.String) { +class StringValueContainer(private val value: String) : ValueContainer() { override fun toValue(): Value = Value.String(value) } @Serializable -class TextValueContainer(private val value: String) : ValueContainer(Type.Text) { +class TextValueContainer(private val value: String) : ValueContainer() { override fun toValue(): Value = Value.Text(value) } @Serializable -class UuidValueContainer(@Serializable(UUIDSerializer::class) private val value: UUID) : ValueContainer(Type.UUID) { +class UuidValueContainer(@Serializable(UUIDSerializer::class) private val value: UUID) : ValueContainer() { override fun toValue(): Value = Value.UUIDValue(value) } @Serializable -class BooleanVectorValueContainer(private val value: BooleanArray) : ValueContainer(Type.BooleanVector(value.size)) { +class BooleanVectorValueContainer(private val value: BooleanArray) : ValueContainer() { override fun toValue(): Value = Value.BooleanVector(value) } @Serializable -class DoubleVectorValueContainer(private val value: DoubleArray) : ValueContainer(Type.DoubleVector(value.size)) { +class DoubleVectorValueContainer(private val value: DoubleArray) : ValueContainer() { override fun toValue(): Value = Value.DoubleVector(value) } @Serializable -class FloatVectorValueContainer(private val value: FloatArray) : ValueContainer(Type.FloatVector(value.size)) { +class FloatVectorValueContainer(private val value: FloatArray) : ValueContainer() { override fun toValue(): Value = Value.FloatVector(value) } @Serializable -class IntVectorValueContainer(private val value: IntArray) : ValueContainer(Type.IntVector(value.size)) { +class IntVectorValueContainer(private val value: IntArray) : ValueContainer() { override fun toValue(): Value = Value.IntVector(value) } @Serializable -class LongVectorValueContainer(private val value: LongArray) : ValueContainer(Type.LongVector(value.size)) { +class LongVectorValueContainer(private val value: LongArray) : ValueContainer() { override fun toValue(): Value = Value.LongVector(value) } \ No newline at end of file diff --git a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlReader.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlReader.kt index 272096b1..35ee11bb 100644 --- a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlReader.kt +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlReader.kt @@ -46,7 +46,14 @@ class ScalarJsonlReader( } private fun queryFulltext(fulltextQuery: SimpleFulltextQuery): Sequence> { - TODO() + + val queryString = fulltextQuery.value.value + val attributeName = fulltextQuery.attributeName ?: return emptySequence() + + return getAll().filter { descriptor -> + (descriptor.values()[attributeName]!! as Value.String).value.contains(queryString) + } + } private fun queryBoolean(query: SimpleBooleanQuery<*>): Sequence> = diff --git a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlReader.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlReader.kt index 1876e5b8..9aaeba0f 100644 --- a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlReader.kt +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlReader.kt @@ -50,7 +50,14 @@ class StructJsonlReader( } private fun queryFulltext(fulltextQuery: SimpleFulltextQuery): Sequence { - TODO() + + val queryString = fulltextQuery.value.value + val attributeName = fulltextQuery.attributeName ?: return emptySequence() + + return getAll().filter { descriptor -> + (descriptor.values()[attributeName]!! as Value.String).value.contains(queryString) + } + } private fun queryBoolean(query: SimpleBooleanQuery<*>): Sequence = getAll().filter { descriptor -> diff --git a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/vector/VectorJsonlReader.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/vector/VectorJsonlReader.kt index f45ddc04..5f98c352 100644 --- a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/vector/VectorJsonlReader.kt +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/vector/VectorJsonlReader.kt @@ -4,7 +4,9 @@ import org.vitrivr.engine.core.model.descriptor.vector.* import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.query.Query import org.vitrivr.engine.core.model.query.proximity.ProximityQuery +import org.vitrivr.engine.core.model.retrievable.Retrieved import org.vitrivr.engine.core.model.types.Value +import org.vitrivr.engine.core.util.knn.FixedSizePriorityQueue import org.vitrivr.engine.database.jsonl.AbstractJsonlReader import org.vitrivr.engine.database.jsonl.model.AttributeContainerList import org.vitrivr.engine.database.jsonl.JsonlConnection @@ -61,9 +63,44 @@ class VectorJsonlReader( else -> throw UnsupportedOperationException("Query of typ ${query::class} is not supported by this reader.") } - private fun queryProximity(query: ProximityQuery<*>): Sequence> { + + private fun queryAndJoinProximity(query: ProximityQuery<*>): Sequence { + + val queue = knn(query) + TODO() } + private fun queryProximity(query: ProximityQuery<*>): Sequence> = knn(query).asSequence().map { it.first } + + + private fun knn(query: ProximityQuery<*>): FixedSizePriorityQueue, Float>> { + + val queue = FixedSizePriorityQueue(query.k.toInt(), + Comparator, Float>> { p0, p1 -> + p0.second.compareTo(p1.second) //TODO consider direction + }) + + getAll().forEach { descriptor -> + val dist = distance(query, descriptor.vector) + queue.add(descriptor to dist) + } + + return queue + + } + + private fun distance(query: ProximityQuery<*>, vector: Value.Vector<*>): Float { + return when (query.value) { + is Value.FloatVector -> query.distance(query.value as Value.FloatVector, vector as Value.FloatVector) + is Value.DoubleVector -> query.distance( + query.value as Value.DoubleVector, + vector as Value.DoubleVector + ).toFloat() + + else -> error("Unsupported query type ${query.value::class.simpleName}") + } + } + } \ No newline at end of file From 08494ee517ecec340b351171e6bbfc1f79acda67 Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Wed, 31 Jul 2024 10:19:44 +0200 Subject: [PATCH 11/11] Added knn query logic --- .../database/jsonl/AbstractJsonlReader.kt | 18 +++++++-- .../jsonl/scalar/ScalarJsonlReader.kt | 2 + .../jsonl/struct/StructJsonlReader.kt | 3 ++ .../jsonl/vector/VectorJsonlReader.kt | 40 +++++++++++++++---- 4 files changed, 51 insertions(+), 12 deletions(-) diff --git a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/AbstractJsonlReader.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/AbstractJsonlReader.kt index 44151b0e..2d7bf6b6 100644 --- a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/AbstractJsonlReader.kt +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/AbstractJsonlReader.kt @@ -10,6 +10,7 @@ import org.vitrivr.engine.core.model.query.Query import org.vitrivr.engine.core.model.retrievable.RetrievableId import org.vitrivr.engine.core.model.retrievable.Retrieved import org.vitrivr.engine.database.jsonl.model.AttributeContainerList +import org.vitrivr.engine.database.jsonl.retrievable.JsonlRetrievableReader import java.io.BufferedReader import java.io.FileReader import java.io.InputStreamReader @@ -58,6 +59,19 @@ abstract class AbstractJsonlReader( } + override fun queryAndJoin(query: Query): Sequence { + val results = query(query).toList() + val ids = results.mapNotNull { it.retrievableId } + + val retrievables = connection.getRetrievableReader().getAll(ids).associateBy { it.id } + + return results.map { descriptor -> + val retrieved = retrievables[descriptor.retrievableId]!! + retrieved.addDescriptor(descriptor) + retrieved as Retrieved + }.asSequence() + } + override fun getForRetrievable(retrievableId: RetrievableId): Sequence { return getAll().filter { it.retrievableId == retrievableId} } @@ -68,10 +82,6 @@ abstract class AbstractJsonlReader( } - override fun queryAndJoin(query: Query): Sequence { - TODO("Not yet implemented") - } - override fun count(): Long { return BufferedReader(InputStreamReader(path.inputStream())).lineSequence().count().toLong() } diff --git a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlReader.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlReader.kt index 35ee11bb..3f9eb154 100644 --- a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlReader.kt +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlReader.kt @@ -6,6 +6,7 @@ import org.vitrivr.engine.core.model.query.Query import org.vitrivr.engine.core.model.query.basics.ComparisonOperator import org.vitrivr.engine.core.model.query.bool.SimpleBooleanQuery import org.vitrivr.engine.core.model.query.fulltext.SimpleFulltextQuery +import org.vitrivr.engine.core.model.retrievable.Retrieved import org.vitrivr.engine.core.model.types.Value import org.vitrivr.engine.database.jsonl.AbstractJsonlReader import org.vitrivr.engine.database.jsonl.model.AttributeContainerList @@ -45,6 +46,7 @@ class ScalarJsonlReader( else -> throw UnsupportedOperationException("The provided query type ${query::class.simpleName} is not supported by this reader.") } + private fun queryFulltext(fulltextQuery: SimpleFulltextQuery): Sequence> { val queryString = fulltextQuery.value.value diff --git a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlReader.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlReader.kt index 9aaeba0f..4bdcbc96 100644 --- a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlReader.kt +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlReader.kt @@ -6,12 +6,14 @@ import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.query.Query import org.vitrivr.engine.core.model.query.bool.SimpleBooleanQuery import org.vitrivr.engine.core.model.query.fulltext.SimpleFulltextQuery +import org.vitrivr.engine.core.model.retrievable.Retrieved import org.vitrivr.engine.core.model.types.Value import org.vitrivr.engine.database.jsonl.AbstractJsonlReader import org.vitrivr.engine.database.jsonl.JsonlConnection import org.vitrivr.engine.database.jsonl.JsonlConnection.Companion.DESCRIPTOR_ID_COLUMN_NAME import org.vitrivr.engine.database.jsonl.JsonlConnection.Companion.RETRIEVABLE_ID_COLUMN_NAME import org.vitrivr.engine.database.jsonl.model.AttributeContainerList +import org.vitrivr.engine.database.jsonl.retrievable.JsonlRetrievableReader import kotlin.reflect.full.primaryConstructor class StructJsonlReader( @@ -49,6 +51,7 @@ class StructJsonlReader( else -> throw UnsupportedOperationException("The provided query type ${query::class.simpleName} is not supported by this reader.") } + private fun queryFulltext(fulltextQuery: SimpleFulltextQuery): Sequence { val queryString = fulltextQuery.value.value diff --git a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/vector/VectorJsonlReader.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/vector/VectorJsonlReader.kt index 5f98c352..79aae4f9 100644 --- a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/vector/VectorJsonlReader.kt +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/vector/VectorJsonlReader.kt @@ -3,8 +3,10 @@ package org.vitrivr.engine.database.jsonl.vector import org.vitrivr.engine.core.model.descriptor.vector.* import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.query.Query +import org.vitrivr.engine.core.model.query.basics.SortOrder import org.vitrivr.engine.core.model.query.proximity.ProximityQuery import org.vitrivr.engine.core.model.retrievable.Retrieved +import org.vitrivr.engine.core.model.retrievable.attributes.DistanceAttribute import org.vitrivr.engine.core.model.types.Value import org.vitrivr.engine.core.util.knn.FixedSizePriorityQueue import org.vitrivr.engine.database.jsonl.AbstractJsonlReader @@ -21,8 +23,8 @@ class VectorJsonlReader( override fun toDescriptor(list: AttributeContainerList): VectorDescriptor<*> { val map = list.list.associateBy { it.attribute.name } - val retrievableId = (map[DESCRIPTOR_ID_COLUMN_NAME]?.value!!.toValue() as Value.UUIDValue).value - val descriptorId = (map[RETRIEVABLE_ID_COLUMN_NAME]?.value!!.toValue() as Value.UUIDValue).value + val retrievableId = (map[RETRIEVABLE_ID_COLUMN_NAME]?.value!!.toValue() as Value.UUIDValue).value + val descriptorId = (map[DESCRIPTOR_ID_COLUMN_NAME]?.value!!.toValue() as Value.UUIDValue).value val value = map["vector"]?.value!!.toValue() return when (prototype) { @@ -63,23 +65,45 @@ class VectorJsonlReader( else -> throw UnsupportedOperationException("Query of typ ${query::class} is not supported by this reader.") } + override fun queryAndJoin(query: Query): Sequence = when (query) { + is ProximityQuery<*> -> queryAndJoinProximity(query) + else -> throw UnsupportedOperationException("Query of typ ${query::class} is not supported by this reader.") + } - private fun queryAndJoinProximity(query: ProximityQuery<*>): Sequence { + private fun queryAndJoinProximity(query: ProximityQuery<*>): Sequence { val queue = knn(query) - TODO() + val ids = queue.mapNotNull { it.first.retrievableId } + + val retrievables = connection.getRetrievableReader().getAll(ids).associateBy { it.id } + + return queue.map { + val retrieved = retrievables[it.first.retrievableId]!! + retrieved.addDescriptor(it.first) + retrieved.addAttribute(DistanceAttribute(it.second)) + retrieved as Retrieved + }.asSequence() + } - private fun queryProximity(query: ProximityQuery<*>): Sequence> = knn(query).asSequence().map { it.first } + private fun queryProximity(query: ProximityQuery<*>): Sequence> = + knn(query).asSequence().map { it.first } private fun knn(query: ProximityQuery<*>): FixedSizePriorityQueue, Float>> { val queue = FixedSizePriorityQueue(query.k.toInt(), - Comparator, Float>> { p0, p1 -> - p0.second.compareTo(p1.second) //TODO consider direction - }) + when (query.order) { + SortOrder.ASC -> Comparator, Float>> { p0, p1 -> + p0.second.compareTo(p1.second) + } + + SortOrder.DESC -> Comparator { p0, p1 -> + p1.second.compareTo(p0.second) + } + } + ) getAll().forEach { descriptor -> val dist = distance(query, descriptor.vector)