diff --git a/settings.gradle b/settings.gradle index 0ad7ee17..62164786 100644 --- a/settings.gradle +++ b/settings.gradle @@ -8,6 +8,7 @@ include 'vitrivr-engine-index' include 'vitrivr-engine-query' include 'vitrivr-engine-server' include 'vitrivr-engine-module-cottontaildb' +include 'vitrivr-engine-module-jsonl' include 'vitrivr-engine-module-pgvector' include 'vitrivr-engine-module-features' include 'vitrivr-engine-module-m3d' diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/metadata/source/exif/ExifMetadataExtractor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/metadata/source/exif/ExifMetadataExtractor.kt index 71851c5a..19c8f7a2 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/metadata/source/exif/ExifMetadataExtractor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/metadata/source/exif/ExifMetadataExtractor.kt @@ -65,6 +65,7 @@ private fun convertType(directory: Directory, tagType: Int, type: Type): Value<* Type.Short -> Value.Short(directory.getObject(tagType) as Short) Type.String -> Value.String(directory.getString(tagType)) Type.Text -> Value.String(directory.getString(tagType)) + Type.UUID -> Value.UUIDValue(UUID.fromString(directory.getString(tagType))) is Type.BooleanVector -> throw IllegalArgumentException("Unsupported type: $type") is Type.DoubleVector -> throw IllegalArgumentException("Unsupported type: $type") is Type.FloatVector -> throw IllegalArgumentException("Unsupported type: $type") diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/Attribute.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/Attribute.kt index df1a6930..051a0b43 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/Attribute.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/Attribute.kt @@ -1,5 +1,6 @@ package org.vitrivr.engine.core.model.descriptor +import kotlinx.serialization.Serializable import org.vitrivr.engine.core.model.types.Type /** The name of an attribute. */ @@ -12,4 +13,5 @@ typealias AttributeName = String * @author Ralph Gasser * @version 1.0.0 */ +@Serializable data class Attribute(val name: AttributeName, val type: Type, val nullable: Boolean = false) \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/Descriptor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/Descriptor.kt index bfbe2939..3a64f459 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/Descriptor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/Descriptor.kt @@ -1,14 +1,16 @@ package org.vitrivr.engine.core.model.descriptor +import kotlinx.serialization.Serializable import org.vitrivr.engine.core.model.Persistable import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.retrievable.Retrievable import org.vitrivr.engine.core.model.retrievable.RetrievableId +import org.vitrivr.engine.core.model.serializer.UUIDSerializer import org.vitrivr.engine.core.model.types.Value import java.util.* /** A typealias to identify the [UUID] identifying a [Descriptor]. */ -typealias DescriptorId = UUID +typealias DescriptorId = @Serializable(UUIDSerializer::class) UUID /** * A [Persistable] [Descriptor] that can be used to describe a [Retrievable]. diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/FloatDescriptor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/FloatDescriptor.kt index b87fa80c..00efb021 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/FloatDescriptor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/FloatDescriptor.kt @@ -1,5 +1,6 @@ package org.vitrivr.engine.core.model.descriptor.scalar +import kotlinx.serialization.Serializable import org.vitrivr.engine.core.model.descriptor.Attribute import org.vitrivr.engine.core.model.descriptor.DescriptorId import org.vitrivr.engine.core.model.descriptor.scalar.ScalarDescriptor.Companion.VALUE_ATTRIBUTE_NAME @@ -14,7 +15,6 @@ import org.vitrivr.engine.core.model.types.Value * @author Ralph Gasser * @version 1.0.0 */ - data class FloatDescriptor( override var id: DescriptorId, override var retrievableId: RetrievableId?, diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/LongDescriptor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/LongDescriptor.kt index a472150d..6e6fb06d 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/LongDescriptor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/LongDescriptor.kt @@ -1,5 +1,6 @@ package org.vitrivr.engine.core.model.descriptor.scalar + import org.vitrivr.engine.core.model.descriptor.Attribute import org.vitrivr.engine.core.model.descriptor.DescriptorId import org.vitrivr.engine.core.model.descriptor.scalar.ScalarDescriptor.Companion.VALUE_ATTRIBUTE_NAME @@ -14,7 +15,6 @@ import org.vitrivr.engine.core.model.types.Value * @author Ralph Gasser * @version 1.0.0 */ - data class LongDescriptor( override var id: DescriptorId, override var retrievableId: RetrievableId?, diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/ScalarDescriptor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/ScalarDescriptor.kt index 452a369f..508dd38b 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/ScalarDescriptor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/ScalarDescriptor.kt @@ -1,5 +1,6 @@ package org.vitrivr.engine.core.model.descriptor.scalar +import kotlinx.serialization.Serializable import org.vitrivr.engine.core.model.descriptor.AttributeName import org.vitrivr.engine.core.model.descriptor.Descriptor import org.vitrivr.engine.core.model.types.Value @@ -10,6 +11,7 @@ import org.vitrivr.engine.core.model.types.Value * @author Ralph Gasser * @version 1.1.0 */ +@Serializable sealed interface ScalarDescriptor> : Descriptor { companion object { diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/query/basics/ComparisonOperator.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/query/basics/ComparisonOperator.kt index ae3a0d37..498062bb 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/query/basics/ComparisonOperator.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/query/basics/ComparisonOperator.kt @@ -1,32 +1,135 @@ package org.vitrivr.engine.core.model.query.basics import org.vitrivr.engine.core.model.query.bool.SimpleBooleanQuery +import org.vitrivr.engine.core.model.types.Value +import java.util.* /** * Enumeration of comparison operators used by the [SimpleBooleanQuery]. * * @author Ralph Gasser * @author Loris Sauter - * @version 1.1.0 + * @author Luca Rossetto + * @version 1.2.0 */ enum class ComparisonOperator(val value: String) { - EQ("=="), - NEQ("!="), - LE("<"), - GR(">"), - LEQ("<="), - GEQ(">="), - LIKE("~="); - - companion object{ + EQ("==") { + override fun compare(v1: Value, v2: Value<*>): Boolean = v1.value == v2.value + }, + NEQ("!=") { + override fun compare(v1: Value, v2: Value<*>): Boolean = v1.value != v2.value + }, + LE("<") { + override fun compare(v1: Value, v2: Value<*>): Boolean = + when (v1) { + is Value.String -> v1.value < (v2.value as String) + is Value.Boolean -> v1.value < (v2.value as Boolean) + is Value.Byte -> v1.value < (v2.value as Byte) + is Value.DateTime -> v1.value < (v2.value as Date) + is Value.Double -> v1.value < (v2.value as Double) + is Value.Float -> v1.value < (v2.value as Float) + is Value.Int -> v1.value < (v2.value as Int) + is Value.Long -> v1.value < (v2.value as Long) + is Value.Short -> v1.value < (v2.value as Short) + is Value.Text -> v1.value < (v2.value as String) + is Value.UUIDValue -> v1.value < (v2.value as UUID) + is Value.BooleanVector, + is Value.DoubleVector, + is Value.FloatVector, + is Value.IntVector, + is Value.LongVector -> false + } + + }, + GR(">") { + override fun compare(v1: Value, v2: Value<*>): Boolean = + when (v1) { + is Value.String -> v1.value > (v2.value as String) + is Value.Boolean -> v1.value > (v2.value as Boolean) + is Value.Byte -> v1.value > (v2.value as Byte) + is Value.DateTime -> v1.value > (v2.value as Date) + is Value.Double -> v1.value > (v2.value as Double) + is Value.Float -> v1.value > (v2.value as Float) + is Value.Int -> v1.value > (v2.value as Int) + is Value.Long -> v1.value > (v2.value as Long) + is Value.Short -> v1.value > (v2.value as Short) + is Value.Text -> v1.value > (v2.value as String) + is Value.UUIDValue -> v1.value > (v2.value as UUID) + is Value.BooleanVector, + is Value.DoubleVector, + is Value.FloatVector, + is Value.IntVector, + is Value.LongVector -> false + } + + }, + LEQ("<=") { + override fun compare(v1: Value, v2: Value<*>): Boolean = + when (v1) { + is Value.String -> v1.value <= (v2.value as String) + is Value.Boolean -> v1.value <= (v2.value as Boolean) + is Value.Byte -> v1.value <= (v2.value as Byte) + is Value.DateTime -> v1.value <= (v2.value as Date) + is Value.Double -> v1.value <= (v2.value as Double) + is Value.Float -> v1.value <= (v2.value as Float) + is Value.Int -> v1.value <= (v2.value as Int) + is Value.Long -> v1.value <= (v2.value as Long) + is Value.Short -> v1.value <= (v2.value as Short) + is Value.Text -> v1.value <= (v2.value as String) + is Value.UUIDValue -> v1.value <= (v2.value as UUID) + is Value.BooleanVector, + is Value.DoubleVector, + is Value.FloatVector, + is Value.IntVector, + is Value.LongVector -> false + } + + }, + GEQ(">=") { + override fun compare(v1: Value, v2: Value<*>): Boolean = + when (v1) { + is Value.String -> v1.value >= (v2.value as String) + is Value.Boolean -> v1.value >= (v2.value as Boolean) + is Value.Byte -> v1.value >= (v2.value as Byte) + is Value.DateTime -> v1.value >= (v2.value as Date) + is Value.Double -> v1.value >= (v2.value as Double) + is Value.Float -> v1.value >= (v2.value as Float) + is Value.Int -> v1.value >= (v2.value as Int) + is Value.Long -> v1.value >= (v2.value as Long) + is Value.Short -> v1.value >= (v2.value as Short) + is Value.Text -> v1.value >= (v2.value as String) + is Value.UUIDValue -> v1.value <= (v2.value as UUID) + is Value.BooleanVector, + is Value.DoubleVector, + is Value.FloatVector, + is Value.IntVector, + is Value.LongVector -> false + } + + }, + LIKE("~=") { + override fun compare(v1: Value, v2: Value<*>): Boolean = + when (v1) { + is Value.String, + is Value.Text -> { + (v1.value as String).replace("\\", "\\\\").replace("[", "\\[").replace("]", "\\]") + .replace("*", "\\*").replace("%", "*").toRegex().matches(v2.value as String) + } + + else -> false + } + + }; + + companion object { /** * Resolves a [ComparisonOperator] from the given [String]. * * @param str The [String] which should be one of the [ComparisonOperator] * @throws IllegalArgumentException In case the given string is not one of the defined ones. */ - fun fromString(str: String):ComparisonOperator{ - return when(str.trim()){ + fun fromString(str: String): ComparisonOperator { + return when (str.trim()) { EQ.value -> EQ NEQ.value -> NEQ LE.value -> LE @@ -38,4 +141,7 @@ enum class ComparisonOperator(val value: String) { } } } + + abstract fun compare(v1: Value, v2: Value<*>): Boolean + } diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/retrievable/Retrievable.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/retrievable/Retrievable.kt index c04a2577..86cdc765 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/retrievable/Retrievable.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/retrievable/Retrievable.kt @@ -1,15 +1,17 @@ package org.vitrivr.engine.core.model.retrievable +import kotlinx.serialization.Serializable import org.vitrivr.engine.core.model.Persistable import org.vitrivr.engine.core.model.content.element.ContentElement import org.vitrivr.engine.core.model.descriptor.Descriptor import org.vitrivr.engine.core.model.relationship.Relationship import org.vitrivr.engine.core.model.retrievable.attributes.RetrievableAttribute +import org.vitrivr.engine.core.model.serializer.UUIDSerializer import java.util.* import java.util.function.Predicate /** A typealias to identify the [UUID] identifying a [Retrievable]. */ -typealias RetrievableId = UUID +typealias RetrievableId = @Serializable(UUIDSerializer::class) UUID /** * A [Persistable] and [Retrievable] unit of information stored by vitrivr. diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/serializer/DateSerializer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/serializer/DateSerializer.kt new file mode 100644 index 00000000..7d78a52a --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/serializer/DateSerializer.kt @@ -0,0 +1,14 @@ +package org.vitrivr.engine.core.model.serializer + +import kotlinx.serialization.KSerializer +import kotlinx.serialization.descriptors.PrimitiveKind +import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor +import kotlinx.serialization.encoding.Decoder +import kotlinx.serialization.encoding.Encoder +import java.util.* + +object DateSerializer: KSerializer { + override val descriptor = PrimitiveSerialDescriptor("Date", PrimitiveKind.LONG) + override fun serialize(encoder: Encoder, value: Date) = encoder.encodeLong(value.time) + override fun deserialize(decoder: Decoder): Date = Date(decoder.decodeLong()) +} \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/serializer/DateTimeSerializer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/serializer/DateTimeSerializer.kt new file mode 100644 index 00000000..3a923c02 --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/serializer/DateTimeSerializer.kt @@ -0,0 +1,15 @@ +package org.vitrivr.engine.core.model.serializer + +import kotlinx.serialization.KSerializer +import kotlinx.serialization.descriptors.PrimitiveKind +import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor +import kotlinx.serialization.encoding.Decoder +import kotlinx.serialization.encoding.Encoder +import org.vitrivr.engine.core.model.types.Value +import java.util.* + +object DateTimeSerializer: KSerializer { + override val descriptor = PrimitiveSerialDescriptor("DateTime", PrimitiveKind.LONG) + override fun serialize(encoder: Encoder, value: Value.DateTime) = encoder.encodeLong(value.value.time) + override fun deserialize(decoder: Decoder): Value.DateTime = Value.DateTime(Date(decoder.decodeLong())) +} \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/serializer/UUIDSerializer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/serializer/UUIDSerializer.kt new file mode 100644 index 00000000..8cffbf8c --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/serializer/UUIDSerializer.kt @@ -0,0 +1,14 @@ +package org.vitrivr.engine.core.model.serializer + +import kotlinx.serialization.KSerializer +import kotlinx.serialization.descriptors.PrimitiveKind +import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor +import kotlinx.serialization.encoding.Decoder +import kotlinx.serialization.encoding.Encoder +import java.util.UUID + +object UUIDSerializer: KSerializer { + override val descriptor = PrimitiveSerialDescriptor("UUID", PrimitiveKind.STRING) + override fun serialize(encoder: Encoder, value: UUID) = encoder.encodeString(value.toString()) + override fun deserialize(decoder: Decoder): UUID = UUID.fromString(decoder.decodeString()) +} \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/types/Type.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/types/Type.kt index 8f3928aa..c38410f9 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/types/Type.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/types/Type.kt @@ -1,5 +1,6 @@ package org.vitrivr.engine.core.model.types +import kotlinx.serialization.Serializable import java.util.* /** @@ -9,6 +10,7 @@ import java.util.* * @author Loris Sauter * @version 2.0.0 */ +@Serializable sealed interface Type { companion object { @@ -55,6 +57,7 @@ sealed interface Type { * * Primarily needed, since simple strings and longer texts are treated differently by certain databases. */ + @Serializable data object Text : Type { override val dimensions: kotlin.Int = 1 override fun defaultValue(): Value<*> = Value.String("") @@ -65,6 +68,7 @@ sealed interface Type { * * Primarily needed, since simple strings and longer texts are treated differently by certain databases. */ + @Serializable data object String : Type { override val dimensions: kotlin.Int = 1 override fun defaultValue(): Value<*> = Value.String("") @@ -73,6 +77,7 @@ sealed interface Type { /** * A [Type] that represents a [Boolean] value. */ + @Serializable data object Boolean : Type { override val dimensions: kotlin.Int = 1 override fun defaultValue(): Value<*> = Value.Boolean(false) @@ -81,6 +86,7 @@ sealed interface Type { /** * A [Type] that represents a [Byte] value. */ + @Serializable data object Byte : Type { override val dimensions: kotlin.Int = 1 override fun defaultValue(): Value<*> = Value.Byte(0) @@ -89,6 +95,7 @@ sealed interface Type { /** * A [Type] that represents a [Short] value. */ + @Serializable data object Short : Type { override val dimensions: kotlin.Int = 1 override fun defaultValue(): Value<*> = Value.Short(0) @@ -97,6 +104,7 @@ sealed interface Type { /** * A [Type] that represents a [Int] value. */ + @Serializable data object Int : Type { override val dimensions: kotlin.Int = 1 override fun defaultValue(): Value<*> = Value.Int(0) @@ -105,6 +113,7 @@ sealed interface Type { /** * A [Type] that represents a [Long] value. */ + @Serializable data object Long : Type { override val dimensions: kotlin.Int = 1 override fun defaultValue(): Value<*> = Value.Long(0L) @@ -113,6 +122,7 @@ sealed interface Type { /** * A [Type] that represents a [Float] value. */ + @Serializable data object Float : Type { override val dimensions: kotlin.Int = 1 override fun defaultValue(): Value<*> = Value.Float(0.0f) @@ -121,6 +131,7 @@ sealed interface Type { /** * A [Type] that represents a [Double] value. */ + @Serializable data object Double : Type { override val dimensions: kotlin.Int = 1 override fun defaultValue(): Value<*> = Value.Double(0.0) @@ -129,14 +140,25 @@ sealed interface Type { /** * A [Type] that represents a [Datetime] value. */ + @Serializable data object Datetime : Type { override val dimensions: kotlin.Int = 1 override fun defaultValue(): Value<*> = Value.DateTime(Date()) } + /** + * A [Type] that represents a [UUID] value. + */ + @Serializable + data object UUID : Type { + override val dimensions: kotlin.Int = 1 + override fun defaultValue(): Value<*> = Value.UUIDValue(java.util.UUID(0L, 0L)) + } + /** * A [Type] that represents a [BooleanVector] value. */ + @Serializable data class BooleanVector(override val dimensions: kotlin.Int) : Type { override fun defaultValue(): Value<*> = Value.BooleanVector(this.dimensions) } @@ -144,6 +166,7 @@ sealed interface Type { /** * A [Type] that represents a [IntVector] value. */ + @Serializable data class IntVector(override val dimensions: kotlin.Int) : Type { override fun defaultValue(): Value<*> = Value.IntVector(this.dimensions) } @@ -151,6 +174,7 @@ sealed interface Type { /** * A [Type] that represents a [LongVector] value. */ + @Serializable data class LongVector(override val dimensions: kotlin.Int) : Type { override fun defaultValue(): Value<*> = Value.LongVector(this.dimensions) } @@ -158,6 +182,7 @@ sealed interface Type { /** * A [Type] that represents a [FloatVector] value. */ + @Serializable data class FloatVector(override val dimensions: kotlin.Int) : Type { override fun defaultValue(): Value<*> = Value.FloatVector(this.dimensions) } @@ -165,6 +190,7 @@ sealed interface Type { /** * A [Type] that represents a [DoubleVector] value. */ + @Serializable data class DoubleVector(override val dimensions: kotlin.Int) : Type { override fun defaultValue(): Value<*> = Value.DoubleVector(this.dimensions) } diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/types/Value.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/types/Value.kt index 5bd1dbbb..65d85950 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/types/Value.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/types/Value.kt @@ -1,5 +1,7 @@ package org.vitrivr.engine.core.model.types +import kotlinx.serialization.Serializable +import org.vitrivr.engine.core.model.serializer.DateTimeSerializer import java.util.* /** @@ -10,6 +12,7 @@ import java.util.* * @author Ralph Gasser * @version 1.0.0 */ +@Serializable() sealed interface Value { companion object { @@ -34,6 +37,7 @@ sealed interface Value { is LongArray -> LongVector(value) is IntArray -> IntVector(value) is Date -> DateTime(value) + is UUID -> UUIDValue(value) else -> throw IllegalArgumentException("Unsupported data type.") } } @@ -41,38 +45,87 @@ sealed interface Value { /** The actual, primitive value. */ val value: T + /** Reference to the [Type] of the Value*/ + val type: Type + sealed interface ScalarValue: Value @JvmInline - value class String(override val value: kotlin.String) : ScalarValue + @Serializable + value class String(override val value: kotlin.String) : ScalarValue { + override val type: Type + get() = Type.String + } @JvmInline - value class Text(override val value: kotlin.String) : ScalarValue + @Serializable + value class Text(override val value: kotlin.String) : ScalarValue { + override val type: Type + get() = Type.Text + } @JvmInline - value class Boolean(override val value: kotlin.Boolean) : ScalarValue + @Serializable + value class Boolean(override val value: kotlin.Boolean) : ScalarValue { + override val type: Type + get() = Type.Boolean + } @JvmInline - value class Byte(override val value: kotlin.Byte) : ScalarValue + @Serializable + value class Byte(override val value: kotlin.Byte) : ScalarValue { + override val type: Type + get() = Type.Byte + } @JvmInline - value class Short(override val value: kotlin.Short) : ScalarValue + @Serializable + value class Short(override val value: kotlin.Short) : ScalarValue { + override val type: Type + get() = Type.Short + } @JvmInline - value class Int(override val value: kotlin.Int) : ScalarValue + @Serializable + value class Int(override val value: kotlin.Int) : ScalarValue { + override val type: Type + get() = Type.Int + } @JvmInline - value class Long(override val value: kotlin.Long) : ScalarValue + @Serializable + value class Long(override val value: kotlin.Long) : ScalarValue { + override val type: Type + get() = Type.Long + } @JvmInline - value class Float(override val value: kotlin.Float) : ScalarValue + @Serializable + value class Float(override val value: kotlin.Float) : ScalarValue { + override val type: Type + get() = Type.Float + } @JvmInline - value class Double(override val value: kotlin.Double) : ScalarValue + @Serializable + value class Double(override val value: kotlin.Double) : ScalarValue { + override val type: Type + get() = Type.Double + } @JvmInline - value class DateTime(override val value: Date) : ScalarValue + @Serializable(with = DateTimeSerializer::class) + value class DateTime(override val value: Date) : ScalarValue { + override val type: Type + get() = Type.Datetime + } + + @JvmInline + value class UUIDValue(override val value: UUID) : ScalarValue { + override val type: Type + get() = Type.UUID + } /** * A [Vector] in vitrivr-engine maps primitive data types. @@ -84,50 +137,71 @@ sealed interface Value { * @property value The actual, primitive value. * @constructor Creates a new [Vector] with the given [value]. */ + @Serializable sealed interface Vector : Value { val size: kotlin.Int } @JvmInline + @Serializable value class BooleanVector(override val value: BooleanArray) : Vector { constructor(size: kotlin.Int, init: (kotlin.Int) -> kotlin.Boolean = { false }) : this(BooleanArray(size, init)) override val size: kotlin.Int get() = this.value.size + + override val type: Type + get() = Type.BooleanVector(size) } @JvmInline + @Serializable value class IntVector(override val value: IntArray) : Vector { constructor(size: kotlin.Int, init: (kotlin.Int) -> kotlin.Int = { 0 }) : this(IntArray(size, init)) override val size: kotlin.Int get() = this.value.size + + override val type: Type + get() = Type.IntVector(size) } @JvmInline + @Serializable value class LongVector(override val value: LongArray) : Vector { constructor(size: kotlin.Int, init: (kotlin.Int) -> kotlin.Long = { 0L }) : this(LongArray(size, init)) override val size: kotlin.Int get() = this.value.size + + override val type: Type + get() = Type.LongVector(size) } @JvmInline + @Serializable value class FloatVector(override val value: FloatArray) : Vector { constructor(size: kotlin.Int, init: (kotlin.Int) -> kotlin.Float = { 0.0f }) : this(FloatArray(size, init)) override val size: kotlin.Int get() = this.value.size + + override val type: Type + get() = Type.FloatVector(size) } @JvmInline + @Serializable value class DoubleVector(override val value: DoubleArray) : Vector { constructor(size: kotlin.Int, init: (kotlin.Int) -> kotlin.Double = { 0.0 }) : this(DoubleArray(size, init)) override val size: kotlin.Int get() = this.value.size + + override val type: Type + get() = Type.DoubleVector(size) } } diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/util/knn/FixedSizePriorityQueue.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/util/knn/FixedSizePriorityQueue.kt new file mode 100644 index 00000000..ccbd0896 --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/util/knn/FixedSizePriorityQueue.kt @@ -0,0 +1,38 @@ +package org.vitrivr.engine.core.util.knn + +import java.util.* + +/** + * Ordered List of fixed size, used for KNN operations + */ + +class FixedSizePriorityQueue(private val maxSize: Int, comparator: Comparator) : TreeSet(comparator) { + + init { + require(maxSize > 0) { "Maximum size must be greater than zero." } + } + + private val elementsLeft: Int + get() = this.maxSize - this.size + + override fun add(element: T): Boolean { + if (elementsLeft > 0) { + // queue isn't full => add element and decrement elementsLeft + val added = super.add(element) + return added + } else { + // there is already 1 or more elements => compare to the least + val compared = super.comparator().compare(this.last(), element) + if (compared > 0) { + // new element is larger than the least in queue => pull the least and add new one to queue + pollLast() + super.add(element) + return true + } else { + // new element is less than the least in queue => return false + return false + } + } + } + +} \ No newline at end of file diff --git a/vitrivr-engine-index/build.gradle b/vitrivr-engine-index/build.gradle index afefe7ef..9efc97a4 100644 --- a/vitrivr-engine-index/build.gradle +++ b/vitrivr-engine-index/build.gradle @@ -13,6 +13,7 @@ dependencies { /** ScrImage (used for image resizing). */ implementation group: 'com.sksamuel.scrimage', name: 'scrimage-core', version: version_scrimage + } /* Publication of vitrivr engine index to Maven Central. */ diff --git a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/Common.kt b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/Common.kt index 310ffae0..6f7a87ec 100644 --- a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/Common.kt +++ b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/Common.kt @@ -61,6 +61,7 @@ internal fun Type.toCottontailType(): Types<*> = when (this) { Type.Float -> Types.Float Type.Double -> Types.Double Type.Datetime -> Types.Date + Type.UUID -> Types.Uuid is Type.BooleanVector -> Types.BooleanVector(this.dimensions) is Type.DoubleVector -> Types.DoubleVector(this.dimensions) is Type.FloatVector -> Types.FloatVector(this.dimensions) @@ -127,6 +128,7 @@ internal fun Value<*>.toCottontailValue(): PublicValue = when (this) { is Value.String -> StringValue(this.value) is Value.Text -> StringValue(this.value) is Value.DateTime -> DateValue(this.value) + is Value.UUIDValue -> UuidValue(this.value) is Value.BooleanVector -> BooleanVectorValue(this.value) is Value.DoubleVector -> DoubleVectorValue(this.value) is Value.FloatVector -> FloatVectorValue(this.value) diff --git a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/struct/StructDescriptorReader.kt b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/struct/StructDescriptorReader.kt index c0df79ec..8a2611a1 100644 --- a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/struct/StructDescriptorReader.kt +++ b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/struct/StructDescriptorReader.kt @@ -74,6 +74,7 @@ class StructDescriptorReader(field: Schema.Field<*, StructDescriptor>, connectio Types.Long -> tuple.asLong(name)?.let { Value.Long(it) } Types.Short -> tuple.asShort(name)?.let { Value.Short(it) } Types.String -> tuple.asString(name)?.let { Value.String(it) } + Types.Uuid -> tuple.asUuidValue(name)?.let { Value.UUIDValue(it.value) } is Types.BooleanVector -> tuple.asBooleanVector(name)?.let { Value.BooleanVector(it) } is Types.DoubleVector -> tuple.asDoubleVector(name)?.let { Value.DoubleVector(it) } is Types.FloatVector -> tuple.asFloatVector(name)?.let { Value.FloatVector(it) } diff --git a/vitrivr-engine-module-jsonl/build.gradle b/vitrivr-engine-module-jsonl/build.gradle new file mode 100644 index 00000000..ef5e253c --- /dev/null +++ b/vitrivr-engine-module-jsonl/build.gradle @@ -0,0 +1,68 @@ +plugins { + id 'org.jetbrains.kotlin.plugin.serialization' version "$version_kotlin" + id 'maven-publish' + id 'signing' +} + +repositories { + mavenCentral() +} + +dependencies { + api project(':vitrivr-engine-core') + + /** vitrivr engine Core is required for running tests. */ + testImplementation(testFixtures(project(':vitrivr-engine-core'))) +} + +/* Publication of vitrivr engine query to Maven Central. */ +publishing { + publications { + mavenJava(MavenPublication) { + groupId = 'org.vitrivr' + artifactId = 'vitrivr-engine-module-jsonl' + version = System.getenv().getOrDefault("MAVEN_PUBLICATION_VERSION", version.toString()) + from components.java + pom { + name = 'vitrivr Engine JSONL Connection Plugin' + description = 'A module that adds a file-based pseudo database using line-wise JSON.' + url = 'https://github.com/vitrivr/vitrivr-engine/' + licenses { + license { + name = 'MIT License' + } + } + developers { + developer { + id = 'ppanopticon' + name = 'Ralph Gasser' + email = 'ralph.gasser@unibas.ch' + } + developer { + id = 'lucaro' + name = 'Luca Rossetto' + email = 'rossetto@ifi.uzh.ch' + } + } + scm { + connection = 'scm:git:https://github.com/vitrivr/vitrivr-engine.git' + url = 'https://github.com/vitrivr/vitrivr-engine/' + } + } + } + } + repositories { + repositories { + maven { + def releasesRepoUrl = 'https://oss.sonatype.org/service/local/staging/deploy/maven2/' + def snapshotsRepoUrl = 'https://oss.sonatype.org/content/repositories/snapshots/' + name = "OSSRH" + url = version.endsWith('SNAPSHOT') ? snapshotsRepoUrl : releasesRepoUrl + credentials { + username = System.getenv("MAVEN_USERNAME") + password = System.getenv("MAVEN_PASSWORD") + } + } + } + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/AbstractJsonlProvider.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/AbstractJsonlProvider.kt new file mode 100644 index 00000000..25d5fad2 --- /dev/null +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/AbstractJsonlProvider.kt @@ -0,0 +1,23 @@ +package org.vitrivr.engine.database.jsonl + +import org.vitrivr.engine.core.database.Connection +import org.vitrivr.engine.core.database.descriptor.DescriptorInitializer +import org.vitrivr.engine.core.database.descriptor.DescriptorProvider +import org.vitrivr.engine.core.database.descriptor.DescriptorWriter +import org.vitrivr.engine.core.model.descriptor.Descriptor +import org.vitrivr.engine.core.model.metamodel.Schema + +abstract class AbstractJsonlProvider : DescriptorProvider { + + override fun newInitializer(connection: Connection, field: Schema.Field<*, D>): DescriptorInitializer = + JsonlInitializer( + field, + connection as JsonlConnection + ) + + override fun newWriter(connection: Connection, field: Schema.Field<*, D>): DescriptorWriter = + JsonlWriter( + field, + connection as JsonlConnection + ) +} \ No newline at end of file diff --git a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/AbstractJsonlReader.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/AbstractJsonlReader.kt new file mode 100644 index 00000000..2d7bf6b6 --- /dev/null +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/AbstractJsonlReader.kt @@ -0,0 +1,88 @@ +package org.vitrivr.engine.database.jsonl + +import kotlinx.serialization.SerializationException +import kotlinx.serialization.json.Json +import org.vitrivr.engine.core.database.descriptor.DescriptorReader +import org.vitrivr.engine.core.model.descriptor.Descriptor +import org.vitrivr.engine.core.model.descriptor.DescriptorId +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.query.Query +import org.vitrivr.engine.core.model.retrievable.RetrievableId +import org.vitrivr.engine.core.model.retrievable.Retrieved +import org.vitrivr.engine.database.jsonl.model.AttributeContainerList +import org.vitrivr.engine.database.jsonl.retrievable.JsonlRetrievableReader +import java.io.BufferedReader +import java.io.FileReader +import java.io.InputStreamReader +import kotlin.io.path.inputStream + +abstract class AbstractJsonlReader( + final override val field: Schema.Field<*, D>, + final override val connection: JsonlConnection +) : + DescriptorReader { + + /** Prototype used to create new instances. */ + protected val prototype = this.field.analyser.prototype(this.field) + + private val path = connection.getPath(field) + + protected abstract fun toDescriptor(list: AttributeContainerList): D + + override fun exists(descriptorId: DescriptorId): Boolean { + return getAll().any { it.id == descriptorId } + } + + override fun get(descriptorId: DescriptorId): D? { + return getAll().firstOrNull { it.id == descriptorId } + } + + override fun getAll(descriptorIds: Iterable): Sequence { + val ids = descriptorIds.toSet() + return getAll().filter { ids.contains(it.id) } + } + + override fun getAll(): Sequence { + + return BufferedReader(InputStreamReader(path.inputStream())).lineSequence().mapNotNull { + try { + val list = Json.decodeFromString(it) + return@mapNotNull toDescriptor(list) + } catch (se: SerializationException) { + LOGGER.error(se) { "Error during deserialization" } + null + } catch (ie: IllegalArgumentException) { + LOGGER.error(ie) { "Error during deserialization" } + null + } + } + + } + + override fun queryAndJoin(query: Query): Sequence { + val results = query(query).toList() + val ids = results.mapNotNull { it.retrievableId } + + val retrievables = connection.getRetrievableReader().getAll(ids).associateBy { it.id } + + return results.map { descriptor -> + val retrieved = retrievables[descriptor.retrievableId]!! + retrieved.addDescriptor(descriptor) + retrieved as Retrieved + }.asSequence() + } + + override fun getForRetrievable(retrievableId: RetrievableId): Sequence { + return getAll().filter { it.retrievableId == retrievableId} + } + + override fun getAllForRetrievable(retrievableIds: Iterable): Sequence { + val ids = retrievableIds.toSet() + return getAll().filter { ids.contains(it.retrievableId) } + } + + + override fun count(): Long { + return BufferedReader(InputStreamReader(path.inputStream())).lineSequence().count().toLong() + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnection.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnection.kt new file mode 100644 index 00000000..8cf018bd --- /dev/null +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnection.kt @@ -0,0 +1,57 @@ +package org.vitrivr.engine.database.jsonl + +import io.github.oshai.kotlinlogging.KotlinLogging.logger +import org.vitrivr.engine.core.database.AbstractConnection +import org.vitrivr.engine.core.database.retrievable.RetrievableInitializer +import org.vitrivr.engine.core.database.retrievable.RetrievableReader +import org.vitrivr.engine.core.database.retrievable.RetrievableWriter +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.database.jsonl.retrievable.JsonlRetrievableInitializer +import org.vitrivr.engine.database.jsonl.retrievable.JsonlRetrievableReader +import org.vitrivr.engine.database.jsonl.retrievable.JsonlRetrievableWriter +import java.nio.file.Path +import kotlin.io.path.absolutePathString + +internal val LOGGER = logger("org.vitrivr.engine.database.jsonl.JsonlConnection") + +class JsonlConnection( + override val schemaName: String, + connectionProvider: JsonlConnectionProvider, + private val root: Path +) : AbstractConnection(schemaName, connectionProvider) { + + override fun withTransaction(action: (Unit) -> T): T { + LOGGER.warn { "Transactions are not supported by the JsonlConnection" } + return action(Unit) + } + + internal val schemaRoot = root.resolve(schemaName) + + fun getPath(field: Schema.Field<*, *>) = schemaRoot.resolve("${field.fieldName}.jsonl") + + override fun getRetrievableInitializer(): RetrievableInitializer = JsonlRetrievableInitializer(this) + + private val writer: JsonlRetrievableWriter by lazy { JsonlRetrievableWriter(this) } + + override fun getRetrievableWriter(): RetrievableWriter = writer + + private val reader: JsonlRetrievableReader by lazy { JsonlRetrievableReader(this) } + + override fun getRetrievableReader(): RetrievableReader = reader + + override fun description(): String = "JsonlConnection on '${root.absolutePathString()}'" + + override fun close() { + writer.close() + } + + companion object { + + /** The column name of a retrievable ID. */ + const val RETRIEVABLE_ID_COLUMN_NAME = "retrievableId" + + /** The column name of a descriptor ID. */ + const val DESCRIPTOR_ID_COLUMN_NAME = "descriptorId" + } + +} \ No newline at end of file diff --git a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnectionProvider.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnectionProvider.kt new file mode 100644 index 00000000..c6fbca90 --- /dev/null +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlConnectionProvider.kt @@ -0,0 +1,59 @@ +package org.vitrivr.engine.database.jsonl + +import org.vitrivr.engine.core.database.AbstractConnectionProvider +import org.vitrivr.engine.core.database.Connection +import org.vitrivr.engine.core.model.descriptor.scalar.* +import org.vitrivr.engine.core.model.descriptor.struct.LabelDescriptor +import org.vitrivr.engine.core.model.descriptor.struct.MapStructDescriptor +import org.vitrivr.engine.core.model.descriptor.struct.metadata.MediaDimensionsDescriptor +import org.vitrivr.engine.core.model.descriptor.struct.metadata.Rectangle2DMetadataDescriptor +import org.vitrivr.engine.core.model.descriptor.struct.metadata.TemporalMetadataDescriptor +import org.vitrivr.engine.core.model.descriptor.struct.metadata.source.FileSourceMetadataDescriptor +import org.vitrivr.engine.core.model.descriptor.struct.metadata.source.VideoSourceMetadataDescriptor +import org.vitrivr.engine.core.model.descriptor.vector.* +import org.vitrivr.engine.database.jsonl.scalar.ScalarJsonlProvider +import org.vitrivr.engine.database.jsonl.struct.StructJsonlProvider +import org.vitrivr.engine.database.jsonl.vector.VectorJsonlProvider +import java.nio.file.Path + + +class JsonlConnectionProvider() : AbstractConnectionProvider() { + + override val databaseName = "JSONL File Connection" + override val version = "1.0.0" + + override fun openConnection(schemaName: String, parameters: Map): Connection { + + val rootPath = parameters["root"] ?: "." + + return JsonlConnection(schemaName, this, Path.of(rootPath)) + + } + + override fun initialize() { + /* Scalar descriptors. */ + this.register(BooleanDescriptor::class, ScalarJsonlProvider) + this.register(IntDescriptor::class, ScalarJsonlProvider) + this.register(LongDescriptor::class, ScalarJsonlProvider) + this.register(FloatDescriptor::class, ScalarJsonlProvider) + this.register(DoubleDescriptor::class, ScalarJsonlProvider) + this.register(StringDescriptor::class, ScalarJsonlProvider) + + /* Vector descriptors. */ + this.register(BooleanVectorDescriptor::class, VectorJsonlProvider) + this.register(IntVectorDescriptor::class, VectorJsonlProvider) + this.register(LongVectorDescriptor::class, VectorJsonlProvider) + this.register(FloatVectorDescriptor::class, VectorJsonlProvider) + this.register(DoubleVectorDescriptor::class, VectorJsonlProvider) + + /* Struct descriptor. */ + this.register(LabelDescriptor::class, StructJsonlProvider) + this.register(FileSourceMetadataDescriptor::class, StructJsonlProvider) + this.register(VideoSourceMetadataDescriptor::class, StructJsonlProvider) + this.register(TemporalMetadataDescriptor::class, StructJsonlProvider) + this.register(Rectangle2DMetadataDescriptor::class, StructJsonlProvider) + this.register(MediaDimensionsDescriptor::class, StructJsonlProvider) + this.register(MapStructDescriptor::class, StructJsonlProvider) + } + +} \ No newline at end of file diff --git a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlInitializer.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlInitializer.kt new file mode 100644 index 00000000..68b7a113 --- /dev/null +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlInitializer.kt @@ -0,0 +1,46 @@ +package org.vitrivr.engine.database.jsonl + +import org.vitrivr.engine.core.database.descriptor.DescriptorInitializer +import org.vitrivr.engine.core.model.descriptor.Descriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import java.io.IOException +import kotlin.io.path.* + +class JsonlInitializer( + override val field: Schema.Field<*, D>, + connection: JsonlConnection +) : DescriptorInitializer { + + private val path = connection.getPath(field) + + override fun initialize() { + try { + path.parent.createDirectories() + path.createFile() + } catch (ioe: IOException) { + LOGGER.error(ioe) { "Cannot initialize '${path.absolutePathString()}'" } + } catch (se: SecurityException) { + LOGGER.error(se) { "Cannot initialize '${path.absolutePathString()}'" } + } + + } + + override fun deinitialize() { + try { + path.deleteIfExists() + } catch (ioe: IOException) { + LOGGER.error(ioe) { "Cannot deinitialize '${path.absolutePathString()}'" } + } + } + + override fun isInitialized(): Boolean = path.exists() + + override fun truncate() { + try { + path.deleteExisting() + path.createFile() + } catch (ioe: IOException) { + LOGGER.error(ioe) { "Cannot truncate '${path.absolutePathString()}'" } + } + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlWriter.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlWriter.kt new file mode 100644 index 00000000..44b5ca44 --- /dev/null +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/JsonlWriter.kt @@ -0,0 +1,83 @@ +package org.vitrivr.engine.database.jsonl + +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json +import org.vitrivr.engine.core.database.descriptor.DescriptorWriter +import org.vitrivr.engine.core.model.descriptor.Attribute +import org.vitrivr.engine.core.model.descriptor.AttributeName +import org.vitrivr.engine.core.model.descriptor.Descriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.types.Type +import org.vitrivr.engine.core.model.types.Value +import org.vitrivr.engine.database.jsonl.JsonlConnection.Companion.DESCRIPTOR_ID_COLUMN_NAME +import org.vitrivr.engine.database.jsonl.JsonlConnection.Companion.RETRIEVABLE_ID_COLUMN_NAME +import org.vitrivr.engine.database.jsonl.model.AttributeContainer +import org.vitrivr.engine.database.jsonl.model.AttributeContainerList +import java.nio.file.StandardOpenOption +import kotlin.io.path.writer + + +class JsonlWriter(override val field: Schema.Field<*, D>, override val connection: JsonlConnection) : + DescriptorWriter, AutoCloseable { + + private val writer = connection.getPath(field).writer(Charsets.UTF_8, StandardOpenOption.APPEND) + + override fun add(item: D): Boolean { + + val valueMap = mutableMapOf?>( + DESCRIPTOR_ID_COLUMN_NAME to Value.UUIDValue(item.id), + RETRIEVABLE_ID_COLUMN_NAME to Value.UUIDValue( + item.retrievableId + ?: throw IllegalArgumentException("A struct descriptor must be associated with a retrievable ID.") + ) + ) + + valueMap.putAll(item.values()) + val attributes = mutableListOf( + Attribute(DESCRIPTOR_ID_COLUMN_NAME, Type.UUID, false), + Attribute(RETRIEVABLE_ID_COLUMN_NAME, Type.UUID, false) + ) + attributes.addAll(item.layout()) + + val list = AttributeContainerList( + attributes.map { attribute -> + AttributeContainer( + attribute, + valueMap[attribute.name] + ) + } + ) + + + writer.write(Json.encodeToString(list)) + writer.write("\n") + writer.flush() + + return true + + } + + override fun addAll(items: Iterable): Boolean { + items.forEach { add(it) } + return true + } + + override fun update(item: D): Boolean { + LOGGER.warn { "JsonlWriter.update is not supported" } + return false + } + + override fun delete(item: D): Boolean { + LOGGER.warn { "JsonlWriter.delete is not supported" } + return false + } + + override fun deleteAll(items: Iterable): Boolean { + LOGGER.warn { "JsonlWriter.deleteAll is not supported" } + return false + } + + override fun close() { + writer.close() + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/AttributeContainer.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/AttributeContainer.kt new file mode 100644 index 00000000..9c83cbd4 --- /dev/null +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/AttributeContainer.kt @@ -0,0 +1,13 @@ +package org.vitrivr.engine.database.jsonl.model + +import kotlinx.serialization.Serializable +import org.vitrivr.engine.core.model.descriptor.Attribute +import org.vitrivr.engine.core.model.types.Value + +@Serializable +data class AttributeContainer(val attribute: Attribute, val value: ValueContainer?) { + constructor(attribute: Attribute, value: Value<*>?) : this( + attribute, + if (value == null) null else ValueContainer.fromValue(value) + ) +} diff --git a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/AttributeContainerList.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/AttributeContainerList.kt new file mode 100644 index 00000000..ae226f9a --- /dev/null +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/AttributeContainerList.kt @@ -0,0 +1,6 @@ +package org.vitrivr.engine.database.jsonl.model + +import kotlinx.serialization.Serializable + +@Serializable +data class AttributeContainerList(val list: List) diff --git a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/JsonlRelationship.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/JsonlRelationship.kt new file mode 100644 index 00000000..efffd87e --- /dev/null +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/JsonlRelationship.kt @@ -0,0 +1,13 @@ +package org.vitrivr.engine.database.jsonl.model + +import kotlinx.serialization.Serializable +import org.vitrivr.engine.core.model.relationship.Relationship +import org.vitrivr.engine.core.model.retrievable.RetrievableId + +@Serializable +data class JsonlRelationship(val sub: RetrievableId, val pred: String, val obj: RetrievableId) { + constructor(relationship: Relationship) : this(relationship.subjectId, relationship.predicate, relationship.objectId) + + fun toTriple() : Triple = Triple(sub, pred, obj) + +} diff --git a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/JsonlRetrievable.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/JsonlRetrievable.kt new file mode 100644 index 00000000..0e9ac6d0 --- /dev/null +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/JsonlRetrievable.kt @@ -0,0 +1,13 @@ +package org.vitrivr.engine.database.jsonl.model + +import kotlinx.serialization.Serializable +import org.vitrivr.engine.core.model.retrievable.Retrievable +import org.vitrivr.engine.core.model.retrievable.RetrievableId +import org.vitrivr.engine.core.model.retrievable.Retrieved + +@Serializable +data class JsonlRetrievable(val id: RetrievableId, val type: String) { + fun toRetrieved(): Retrieved = Retrieved(id, if (type.isEmpty()) null else type, false) + + constructor(retrievable: Retrievable) : this(retrievable.id, retrievable.type ?: "") +} diff --git a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/ValueContainer.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/ValueContainer.kt new file mode 100644 index 00000000..aa135690 --- /dev/null +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/model/ValueContainer.kt @@ -0,0 +1,116 @@ +package org.vitrivr.engine.database.jsonl.model + +import kotlinx.serialization.Serializable +import org.vitrivr.engine.core.model.serializer.DateSerializer +import org.vitrivr.engine.core.model.serializer.UUIDSerializer +import org.vitrivr.engine.core.model.types.Value +import java.util.* + +@Serializable +sealed class ValueContainer { + + companion object { + fun fromValue(value: Value<*>): ValueContainer = when (value) { + is Value.Boolean -> BooleanValueContainer(value.value) + is Value.Byte -> ByteValueContainer(value.value) + is Value.DateTime -> DateTimeValueContainer(value.value) + is Value.Double -> DoubleValueContainer(value.value) + is Value.Float -> FloatValueContainer(value.value) + is Value.Int -> IntValueContainer(value.value) + is Value.Long -> LongValueContainer(value.value) + is Value.Short -> ShortValueContainer(value.value) + is Value.String -> StringValueContainer(value.value) + is Value.Text -> TextValueContainer(value.value) + is Value.UUIDValue -> UuidValueContainer(value.value) + is Value.BooleanVector -> BooleanVectorValueContainer(value.value) + is Value.DoubleVector -> DoubleVectorValueContainer(value.value) + is Value.FloatVector -> FloatVectorValueContainer(value.value) + is Value.IntVector -> IntVectorValueContainer(value.value) + is Value.LongVector -> LongVectorValueContainer(value.value) + } + } + + abstract fun toValue(): Value<*> + +} + +@Serializable +class BooleanValueContainer(private val value: Boolean) : ValueContainer() { + override fun toValue(): Value = Value.Boolean(value) +} + +@Serializable +class ByteValueContainer(private val value: Byte) : ValueContainer() { + override fun toValue(): Value = Value.Byte(value) +} + +@Serializable +class DateTimeValueContainer(@Serializable(DateSerializer::class) private val value: Date) : + ValueContainer() { + override fun toValue(): Value = Value.DateTime(value) +} + +@Serializable +class DoubleValueContainer(private val value: Double) : ValueContainer() { + override fun toValue(): Value = Value.Double(value) +} + +@Serializable +class FloatValueContainer(private val value: Float) : ValueContainer() { + override fun toValue(): Value = Value.Float(value) +} + +@Serializable +class IntValueContainer(private val value: Int) : ValueContainer() { + override fun toValue(): Value = Value.Int(value) +} + +@Serializable +class LongValueContainer(private val value: Long) : ValueContainer() { + override fun toValue(): Value = Value.Long(value) +} + +@Serializable +class ShortValueContainer(private val value: Short) : ValueContainer() { + override fun toValue(): Value = Value.Short(value) +} + +@Serializable +class StringValueContainer(private val value: String) : ValueContainer() { + override fun toValue(): Value = Value.String(value) +} + +@Serializable +class TextValueContainer(private val value: String) : ValueContainer() { + override fun toValue(): Value = Value.Text(value) +} + +@Serializable +class UuidValueContainer(@Serializable(UUIDSerializer::class) private val value: UUID) : ValueContainer() { + override fun toValue(): Value = Value.UUIDValue(value) +} + +@Serializable +class BooleanVectorValueContainer(private val value: BooleanArray) : ValueContainer() { + override fun toValue(): Value = Value.BooleanVector(value) +} + +@Serializable +class DoubleVectorValueContainer(private val value: DoubleArray) : ValueContainer() { + override fun toValue(): Value = Value.DoubleVector(value) +} + +@Serializable +class FloatVectorValueContainer(private val value: FloatArray) : ValueContainer() { + override fun toValue(): Value = Value.FloatVector(value) +} + +@Serializable +class IntVectorValueContainer(private val value: IntArray) : ValueContainer() { + override fun toValue(): Value = Value.IntVector(value) +} + +@Serializable +class LongVectorValueContainer(private val value: LongArray) : ValueContainer() { + override fun toValue(): Value = Value.LongVector(value) +} \ No newline at end of file diff --git a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableInitializer.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableInitializer.kt new file mode 100644 index 00000000..97c46252 --- /dev/null +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableInitializer.kt @@ -0,0 +1,45 @@ +package org.vitrivr.engine.database.jsonl.retrievable + +import org.vitrivr.engine.core.database.retrievable.RetrievableInitializer +import org.vitrivr.engine.database.jsonl.JsonlConnection +import org.vitrivr.engine.database.jsonl.LOGGER +import java.io.IOException +import kotlin.io.path.* + +class JsonlRetrievableInitializer(private val connection: JsonlConnection) : RetrievableInitializer { + + private val retrievablePath = connection.schemaRoot.resolve("retrievables.jsonl") + private val connectionPath = connection.schemaRoot.resolve("retrievable_connections.jsonl") + + override fun initialize() { + try { + connection.schemaRoot.createDirectories() + retrievablePath.createFile() + connectionPath.createFile() + } catch (ioe: IOException) { + LOGGER.error(ioe) { "Cannot initialize '${connection.schemaRoot.absolutePathString()}'" } + } + } + + override fun deinitialize() { + try { + retrievablePath.deleteIfExists() + connectionPath.deleteIfExists() + } catch (ioe: IOException) { + LOGGER.error(ioe) { "Cannot delete '${connection.schemaRoot.absolutePathString()}'" } + } + } + + override fun isInitialized(): Boolean = retrievablePath.exists() && connectionPath.exists() + + override fun truncate() { + try { + retrievablePath.deleteIfExists() + retrievablePath.createFile() + connectionPath.deleteIfExists() + connectionPath.createFile() + } catch (ioe: IOException) { + LOGGER.error(ioe) { "Cannot truncate '${connection.schemaRoot.absolutePathString()}'" } + } + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableReader.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableReader.kt new file mode 100644 index 00000000..c9500365 --- /dev/null +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableReader.kt @@ -0,0 +1,72 @@ +package org.vitrivr.engine.database.jsonl.retrievable + +import kotlinx.serialization.SerializationException +import kotlinx.serialization.json.Json +import org.vitrivr.engine.core.database.retrievable.RetrievableReader +import org.vitrivr.engine.core.model.retrievable.Retrievable +import org.vitrivr.engine.core.model.retrievable.RetrievableId +import org.vitrivr.engine.database.jsonl.JsonlConnection +import org.vitrivr.engine.database.jsonl.LOGGER +import org.vitrivr.engine.database.jsonl.model.JsonlRelationship +import org.vitrivr.engine.database.jsonl.model.JsonlRetrievable +import java.io.BufferedReader +import java.io.InputStreamReader +import kotlin.io.path.inputStream + +class JsonlRetrievableReader(override val connection: JsonlConnection) : RetrievableReader { + + private val retrievablePath = connection.schemaRoot.resolve("retrievables.jsonl") + private val connectionPath = connection.schemaRoot.resolve("retrievable_connections.jsonl") + + override fun get(id: RetrievableId): Retrievable? = getAll().firstOrNull { it.id == id } + + override fun exists(id: RetrievableId): Boolean = get(id) != null + + override fun getAll(ids: Iterable): Sequence { + val idSet = ids.toSet() + return getAll().filter { idSet.contains(it.id) } + } + + override fun getConnections( + subjectIds: Collection, + predicates: Collection, + objectIds: Collection + ): Sequence> { + val subIds = subjectIds.toSet() + val predIds = predicates.toSet() + val objIds = objectIds.toSet() + + return BufferedReader(InputStreamReader(connectionPath.inputStream())).lineSequence().mapNotNull { + try { + Json.decodeFromString(it) + } catch (se: SerializationException) { + LOGGER.error(se) { "Error during deserialization" } + null + } catch (ie: IllegalArgumentException) { + LOGGER.error(ie) { "Error during deserialization" } + null + } + }.filter { + (subIds.isEmpty() || subIds.contains(it.sub)) && + (predIds.isEmpty() || predIds.contains(it.pred)) && + (objIds.isEmpty() || objIds.contains(it.obj)) + }.map { it.toTriple() } + } + + override fun getAll(): Sequence { + return BufferedReader(InputStreamReader(retrievablePath.inputStream())).lineSequence().mapNotNull { + try { + Json.decodeFromString(it).toRetrieved() + } catch (se: SerializationException) { + LOGGER.error(se) { "Error during deserialization" } + null + } catch (ie: IllegalArgumentException) { + LOGGER.error(ie) { "Error during deserialization" } + null + } + } + } + + override fun count(): Long = BufferedReader(InputStreamReader(retrievablePath.inputStream())).lineSequence().count().toLong() + +} \ No newline at end of file diff --git a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableWriter.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableWriter.kt new file mode 100644 index 00000000..fba04529 --- /dev/null +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/retrievable/JsonlRetrievableWriter.kt @@ -0,0 +1,77 @@ +package org.vitrivr.engine.database.jsonl.retrievable + +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json +import org.vitrivr.engine.core.database.retrievable.RetrievableWriter +import org.vitrivr.engine.core.model.relationship.Relationship +import org.vitrivr.engine.core.model.retrievable.Retrievable +import org.vitrivr.engine.database.jsonl.JsonlConnection +import org.vitrivr.engine.database.jsonl.LOGGER +import org.vitrivr.engine.database.jsonl.model.JsonlRelationship +import org.vitrivr.engine.database.jsonl.model.JsonlRetrievable +import java.io.File +import java.io.FileWriter +import java.nio.file.StandardOpenOption +import kotlin.io.path.writer + +class JsonlRetrievableWriter(override val connection: JsonlConnection) : RetrievableWriter, AutoCloseable { + + private val retrievableWriter = connection.schemaRoot.resolve("retrievables.jsonl").writer(Charsets.UTF_8, StandardOpenOption.APPEND) + private val connectionWriter = connection.schemaRoot.resolve("retrievable_connections.jsonl").writer(Charsets.UTF_8, StandardOpenOption.APPEND) + + @Synchronized + override fun connect(relationship: Relationship): Boolean { + connectionWriter.write(Json.encodeToString(JsonlRelationship(relationship))) + connectionWriter.write("\n") + connectionWriter.flush() + return true + } + + override fun connectAll(relationships: Iterable): Boolean { + relationships.forEach { connect(it) } + return true + } + + override fun disconnect(relationship: Relationship): Boolean { + LOGGER.warn { "JsonlRetrievableWriter.disconnect is not supported" } + return false + } + + override fun disconnectAll(relationships: Iterable): Boolean { + LOGGER.warn { "JsonlRetrievableWriter.disconnectAll is not supported" } + return false + } + + @Synchronized + override fun add(item: Retrievable): Boolean { + retrievableWriter.write(Json.encodeToString(JsonlRetrievable(item))) + retrievableWriter.write("\n") + retrievableWriter.flush() + return true + } + + override fun addAll(items: Iterable): Boolean { + items.forEach { add(it) } + return true + } + + override fun update(item: Retrievable): Boolean { + LOGGER.warn { "JsonlRetrievableWriter.update is not supported" } + return false + } + + override fun delete(item: Retrievable): Boolean { + LOGGER.warn { "JsonlRetrievableWriter.delete is not supported" } + return false + } + + override fun deleteAll(items: Iterable): Boolean { + LOGGER.warn { "JsonlRetrievableWriter.deleteAll is not supported" } + return false + } + + override fun close() { + retrievableWriter.close() + connectionWriter.close() + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlProvider.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlProvider.kt new file mode 100644 index 00000000..5e6e9b95 --- /dev/null +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlProvider.kt @@ -0,0 +1,12 @@ +package org.vitrivr.engine.database.jsonl.scalar + +import org.vitrivr.engine.core.database.Connection +import org.vitrivr.engine.core.database.descriptor.DescriptorReader +import org.vitrivr.engine.core.model.descriptor.scalar.ScalarDescriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.database.jsonl.AbstractJsonlProvider +import org.vitrivr.engine.database.jsonl.JsonlConnection + +object ScalarJsonlProvider: AbstractJsonlProvider>() { + override fun newReader(connection: Connection, field: Schema.Field<*, ScalarDescriptor<*>>): DescriptorReader> = ScalarJsonlReader(field, connection as JsonlConnection) +} \ No newline at end of file diff --git a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlReader.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlReader.kt new file mode 100644 index 00000000..3f9eb154 --- /dev/null +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/scalar/ScalarJsonlReader.kt @@ -0,0 +1,65 @@ +package org.vitrivr.engine.database.jsonl.scalar + +import org.vitrivr.engine.core.model.descriptor.scalar.* +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.query.Query +import org.vitrivr.engine.core.model.query.basics.ComparisonOperator +import org.vitrivr.engine.core.model.query.bool.SimpleBooleanQuery +import org.vitrivr.engine.core.model.query.fulltext.SimpleFulltextQuery +import org.vitrivr.engine.core.model.retrievable.Retrieved +import org.vitrivr.engine.core.model.types.Value +import org.vitrivr.engine.database.jsonl.AbstractJsonlReader +import org.vitrivr.engine.database.jsonl.model.AttributeContainerList +import org.vitrivr.engine.database.jsonl.JsonlConnection +import org.vitrivr.engine.database.jsonl.JsonlConnection.Companion.DESCRIPTOR_ID_COLUMN_NAME +import org.vitrivr.engine.database.jsonl.JsonlConnection.Companion.RETRIEVABLE_ID_COLUMN_NAME + +class ScalarJsonlReader( + field: Schema.Field<*, ScalarDescriptor<*>>, + connection: JsonlConnection +) : AbstractJsonlReader>(field, connection) { + + override fun toDescriptor(list: AttributeContainerList): ScalarDescriptor<*> { + + val map = list.list.associateBy { it.attribute.name } + val retrievableId = (map[DESCRIPTOR_ID_COLUMN_NAME]?.value!!.toValue() as Value.UUIDValue).value + val descriptorId = (map[RETRIEVABLE_ID_COLUMN_NAME]?.value!!.toValue() as Value.UUIDValue).value + val value = map["value"]?.value!!.toValue() + + return when (prototype) { + is BooleanDescriptor -> BooleanDescriptor(retrievableId, descriptorId, value as Value.Boolean) + is DoubleDescriptor -> DoubleDescriptor(retrievableId, descriptorId, value as Value.Double) + is FloatDescriptor -> FloatDescriptor(retrievableId, descriptorId, value as Value.Float) + is IntDescriptor -> IntDescriptor(retrievableId, descriptorId, value as Value.Int) + is LongDescriptor -> LongDescriptor(retrievableId, descriptorId, value as Value.Long) + is StringDescriptor -> StringDescriptor(retrievableId, descriptorId, value as Value.String) + else -> { + error("Unsupported type $prototype") + } + } + + } + + override fun query(query: Query): Sequence> = when (query) { + is SimpleFulltextQuery -> this.queryFulltext(query) + is SimpleBooleanQuery<*> -> this.queryBoolean(query) + else -> throw UnsupportedOperationException("The provided query type ${query::class.simpleName} is not supported by this reader.") + } + + + private fun queryFulltext(fulltextQuery: SimpleFulltextQuery): Sequence> { + + val queryString = fulltextQuery.value.value + val attributeName = fulltextQuery.attributeName ?: return emptySequence() + + return getAll().filter { descriptor -> + (descriptor.values()[attributeName]!! as Value.String).value.contains(queryString) + } + + } + + private fun queryBoolean(query: SimpleBooleanQuery<*>): Sequence> = + getAll().filter { descriptor -> + query.comparison.compare(descriptor.value, query.value) + } +} diff --git a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlProvider.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlProvider.kt new file mode 100644 index 00000000..0d9a9b50 --- /dev/null +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlProvider.kt @@ -0,0 +1,12 @@ +package org.vitrivr.engine.database.jsonl.struct + +import org.vitrivr.engine.core.database.Connection +import org.vitrivr.engine.core.database.descriptor.DescriptorReader +import org.vitrivr.engine.core.model.descriptor.struct.StructDescriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.database.jsonl.AbstractJsonlProvider +import org.vitrivr.engine.database.jsonl.JsonlConnection + +object StructJsonlProvider: AbstractJsonlProvider() { + override fun newReader(connection: Connection, field: Schema.Field<*, StructDescriptor>): DescriptorReader = StructJsonlReader(field, connection as JsonlConnection) +} \ No newline at end of file diff --git a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlReader.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlReader.kt new file mode 100644 index 00000000..4bdcbc96 --- /dev/null +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/struct/StructJsonlReader.kt @@ -0,0 +1,71 @@ +package org.vitrivr.engine.database.jsonl.struct + +import org.vitrivr.engine.core.model.descriptor.AttributeName +import org.vitrivr.engine.core.model.descriptor.struct.StructDescriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.query.Query +import org.vitrivr.engine.core.model.query.bool.SimpleBooleanQuery +import org.vitrivr.engine.core.model.query.fulltext.SimpleFulltextQuery +import org.vitrivr.engine.core.model.retrievable.Retrieved +import org.vitrivr.engine.core.model.types.Value +import org.vitrivr.engine.database.jsonl.AbstractJsonlReader +import org.vitrivr.engine.database.jsonl.JsonlConnection +import org.vitrivr.engine.database.jsonl.JsonlConnection.Companion.DESCRIPTOR_ID_COLUMN_NAME +import org.vitrivr.engine.database.jsonl.JsonlConnection.Companion.RETRIEVABLE_ID_COLUMN_NAME +import org.vitrivr.engine.database.jsonl.model.AttributeContainerList +import org.vitrivr.engine.database.jsonl.retrievable.JsonlRetrievableReader +import kotlin.reflect.full.primaryConstructor + +class StructJsonlReader( + field: Schema.Field<*, StructDescriptor>, + connection: JsonlConnection +) : AbstractJsonlReader(field, connection) { + + override fun toDescriptor(list: AttributeContainerList): StructDescriptor { + + val map = list.list.associateBy { it.attribute.name } + val constructor = this.field.analyser.descriptorClass.primaryConstructor + ?: throw IllegalStateException("Provided type ${this.field.analyser.descriptorClass} does not have a primary constructor.") + val valueMap = mutableMapOf>() + + val retrievableId = (map[DESCRIPTOR_ID_COLUMN_NAME]?.value!!.toValue() as Value.UUIDValue).value + val descriptorId = (map[RETRIEVABLE_ID_COLUMN_NAME]?.value!!.toValue() as Value.UUIDValue).value + val parameters: MutableList = mutableListOf( + descriptorId, + retrievableId, + valueMap, + this.field + ) + + prototype.layout().forEach { attribute -> + val value = map[attribute.name]!!.value?.toValue()!! + valueMap[attribute.name] = value + } + + return constructor.call(*parameters.toTypedArray()) + } + + override fun query(query: Query): Sequence = when (query) { + is SimpleFulltextQuery -> this.queryFulltext(query) + is SimpleBooleanQuery<*> -> this.queryBoolean(query) + else -> throw UnsupportedOperationException("The provided query type ${query::class.simpleName} is not supported by this reader.") + } + + + private fun queryFulltext(fulltextQuery: SimpleFulltextQuery): Sequence { + + val queryString = fulltextQuery.value.value + val attributeName = fulltextQuery.attributeName ?: return emptySequence() + + return getAll().filter { descriptor -> + (descriptor.values()[attributeName]!! as Value.String).value.contains(queryString) + } + + } + + private fun queryBoolean(query: SimpleBooleanQuery<*>): Sequence = getAll().filter { descriptor -> + query.comparison.compare(descriptor.values()[query.attributeName!!]!!, query.value) + } + + +} \ No newline at end of file diff --git a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/vector/VectorJsonlProvider.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/vector/VectorJsonlProvider.kt new file mode 100644 index 00000000..4cb88c3c --- /dev/null +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/vector/VectorJsonlProvider.kt @@ -0,0 +1,12 @@ +package org.vitrivr.engine.database.jsonl.vector + +import org.vitrivr.engine.core.database.Connection +import org.vitrivr.engine.core.database.descriptor.DescriptorReader +import org.vitrivr.engine.core.model.descriptor.vector.VectorDescriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.database.jsonl.AbstractJsonlProvider +import org.vitrivr.engine.database.jsonl.JsonlConnection + +object VectorJsonlProvider: AbstractJsonlProvider>() { + override fun newReader(connection: Connection, field: Schema.Field<*, VectorDescriptor<*>>): DescriptorReader> = VectorJsonlReader(field, connection as JsonlConnection) +} \ No newline at end of file diff --git a/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/vector/VectorJsonlReader.kt b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/vector/VectorJsonlReader.kt new file mode 100644 index 00000000..79aae4f9 --- /dev/null +++ b/vitrivr-engine-module-jsonl/src/main/kotlin/org/vitrivr/engine/database/jsonl/vector/VectorJsonlReader.kt @@ -0,0 +1,130 @@ +package org.vitrivr.engine.database.jsonl.vector + +import org.vitrivr.engine.core.model.descriptor.vector.* +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.query.Query +import org.vitrivr.engine.core.model.query.basics.SortOrder +import org.vitrivr.engine.core.model.query.proximity.ProximityQuery +import org.vitrivr.engine.core.model.retrievable.Retrieved +import org.vitrivr.engine.core.model.retrievable.attributes.DistanceAttribute +import org.vitrivr.engine.core.model.types.Value +import org.vitrivr.engine.core.util.knn.FixedSizePriorityQueue +import org.vitrivr.engine.database.jsonl.AbstractJsonlReader +import org.vitrivr.engine.database.jsonl.model.AttributeContainerList +import org.vitrivr.engine.database.jsonl.JsonlConnection +import org.vitrivr.engine.database.jsonl.JsonlConnection.Companion.DESCRIPTOR_ID_COLUMN_NAME +import org.vitrivr.engine.database.jsonl.JsonlConnection.Companion.RETRIEVABLE_ID_COLUMN_NAME + +class VectorJsonlReader( + field: Schema.Field<*, VectorDescriptor<*>>, + connection: JsonlConnection +) : AbstractJsonlReader>(field, connection) { + + override fun toDescriptor(list: AttributeContainerList): VectorDescriptor<*> { + + val map = list.list.associateBy { it.attribute.name } + val retrievableId = (map[RETRIEVABLE_ID_COLUMN_NAME]?.value!!.toValue() as Value.UUIDValue).value + val descriptorId = (map[DESCRIPTOR_ID_COLUMN_NAME]?.value!!.toValue() as Value.UUIDValue).value + val value = map["vector"]?.value!!.toValue() + + return when (prototype) { + is BooleanVectorDescriptor -> BooleanVectorDescriptor( + descriptorId, + retrievableId, + value as Value.BooleanVector + ) + + is FloatVectorDescriptor -> FloatVectorDescriptor( + descriptorId, + retrievableId, + value as Value.FloatVector + ) + + is DoubleVectorDescriptor -> DoubleVectorDescriptor( + descriptorId, + retrievableId, + value as Value.DoubleVector + ) + + is IntVectorDescriptor -> IntVectorDescriptor( + descriptorId, + retrievableId, + value as Value.IntVector + ) + + is LongVectorDescriptor -> LongVectorDescriptor( + descriptorId, + retrievableId, + value as Value.LongVector + ) + } + } + + override fun query(query: Query): Sequence> = when (query) { + is ProximityQuery<*> -> queryProximity(query) + else -> throw UnsupportedOperationException("Query of typ ${query::class} is not supported by this reader.") + } + + override fun queryAndJoin(query: Query): Sequence = when (query) { + is ProximityQuery<*> -> queryAndJoinProximity(query) + else -> throw UnsupportedOperationException("Query of typ ${query::class} is not supported by this reader.") + } + + + private fun queryAndJoinProximity(query: ProximityQuery<*>): Sequence { + val queue = knn(query) + + val ids = queue.mapNotNull { it.first.retrievableId } + + val retrievables = connection.getRetrievableReader().getAll(ids).associateBy { it.id } + + return queue.map { + val retrieved = retrievables[it.first.retrievableId]!! + retrieved.addDescriptor(it.first) + retrieved.addAttribute(DistanceAttribute(it.second)) + retrieved as Retrieved + }.asSequence() + + } + + private fun queryProximity(query: ProximityQuery<*>): Sequence> = + knn(query).asSequence().map { it.first } + + + private fun knn(query: ProximityQuery<*>): FixedSizePriorityQueue, Float>> { + + val queue = FixedSizePriorityQueue(query.k.toInt(), + when (query.order) { + SortOrder.ASC -> Comparator, Float>> { p0, p1 -> + p0.second.compareTo(p1.second) + } + + SortOrder.DESC -> Comparator { p0, p1 -> + p1.second.compareTo(p0.second) + } + } + ) + + getAll().forEach { descriptor -> + val dist = distance(query, descriptor.vector) + queue.add(descriptor to dist) + } + + return queue + + } + + private fun distance(query: ProximityQuery<*>, vector: Value.Vector<*>): Float { + return when (query.value) { + is Value.FloatVector -> query.distance(query.value as Value.FloatVector, vector as Value.FloatVector) + is Value.DoubleVector -> query.distance( + query.value as Value.DoubleVector, + vector as Value.DoubleVector + ).toFloat() + + else -> error("Unsupported query type ${query.value::class.simpleName}") + } + } + + +} \ No newline at end of file diff --git a/vitrivr-engine-module-jsonl/src/main/resources/META-INF/services/org.vitrivr.engine.core.database.ConnectionProvider b/vitrivr-engine-module-jsonl/src/main/resources/META-INF/services/org.vitrivr.engine.core.database.ConnectionProvider new file mode 100644 index 00000000..41cabd09 --- /dev/null +++ b/vitrivr-engine-module-jsonl/src/main/resources/META-INF/services/org.vitrivr.engine.core.database.ConnectionProvider @@ -0,0 +1 @@ +org.vitrivr.engine.database.jsonl.JsonlConnectionProvider \ No newline at end of file diff --git a/vitrivr-engine-module-jsonl/src/test/kotlin/org/vitrivr/engine/database/jsonl/descriptor/DescriptorInitializerTest.kt b/vitrivr-engine-module-jsonl/src/test/kotlin/org/vitrivr/engine/database/jsonl/descriptor/DescriptorInitializerTest.kt new file mode 100644 index 00000000..37222081 --- /dev/null +++ b/vitrivr-engine-module-jsonl/src/test/kotlin/org/vitrivr/engine/database/jsonl/descriptor/DescriptorInitializerTest.kt @@ -0,0 +1,6 @@ +package org.vitrivr.engine.database.jsonl.descriptor + +import org.vitrivr.engine.core.database.descriptor.AbstractDescriptorInitializerTest + + +class DescriptorInitializerTest : AbstractDescriptorInitializerTest("test-schema-jsonl.json") \ No newline at end of file diff --git a/vitrivr-engine-module-jsonl/src/test/kotlin/org/vitrivr/engine/database/jsonl/descriptor/struct/FileMetadataDescriptorReaderTest.kt b/vitrivr-engine-module-jsonl/src/test/kotlin/org/vitrivr/engine/database/jsonl/descriptor/struct/FileMetadataDescriptorReaderTest.kt new file mode 100644 index 00000000..87d1c796 --- /dev/null +++ b/vitrivr-engine-module-jsonl/src/test/kotlin/org/vitrivr/engine/database/jsonl/descriptor/struct/FileMetadataDescriptorReaderTest.kt @@ -0,0 +1,6 @@ +package org.vitrivr.engine.database.jsonl.descriptor.struct + +import org.vitrivr.engine.core.database.descriptor.struct.AbstractFileMetadataDescriptorReaderTest + + +class FileMetadataDescriptorReaderTest : AbstractFileMetadataDescriptorReaderTest("test-schema-jsonl.json") \ No newline at end of file diff --git a/vitrivr-engine-module-jsonl/src/test/kotlin/org/vitrivr/engine/database/jsonl/descriptor/vector/FloatVectorDescriptorReaderTest.kt b/vitrivr-engine-module-jsonl/src/test/kotlin/org/vitrivr/engine/database/jsonl/descriptor/vector/FloatVectorDescriptorReaderTest.kt new file mode 100644 index 00000000..768645f0 --- /dev/null +++ b/vitrivr-engine-module-jsonl/src/test/kotlin/org/vitrivr/engine/database/jsonl/descriptor/vector/FloatVectorDescriptorReaderTest.kt @@ -0,0 +1,6 @@ +package org.vitrivr.engine.database.jsonl.descriptor.vector + +import org.vitrivr.engine.core.database.descriptor.vector.AbstractFloatVectorDescriptorReaderTest + + +class FloatVectorDescriptorReaderTest : AbstractFloatVectorDescriptorReaderTest("test-schema-jsonl.json") \ No newline at end of file diff --git a/vitrivr-engine-module-jsonl/src/test/kotlin/org/vitrivr/engine/database/jsonl/retrievable/RetrievableInitializerTest.kt b/vitrivr-engine-module-jsonl/src/test/kotlin/org/vitrivr/engine/database/jsonl/retrievable/RetrievableInitializerTest.kt new file mode 100644 index 00000000..872b96dd --- /dev/null +++ b/vitrivr-engine-module-jsonl/src/test/kotlin/org/vitrivr/engine/database/jsonl/retrievable/RetrievableInitializerTest.kt @@ -0,0 +1,8 @@ +package org.vitrivr.engine.database.jsonl.retrievable + +import org.vitrivr.engine.core.database.retrievable.AbstractRetrievableInitializerTest + + +class RetrievableInitializerTest : AbstractRetrievableInitializerTest("test-schema-jsonl.json") { + +} \ No newline at end of file diff --git a/vitrivr-engine-module-jsonl/src/test/kotlin/org/vitrivr/engine/database/jsonl/retrievable/RetrievableWriterTest.kt b/vitrivr-engine-module-jsonl/src/test/kotlin/org/vitrivr/engine/database/jsonl/retrievable/RetrievableWriterTest.kt new file mode 100644 index 00000000..acaac7a1 --- /dev/null +++ b/vitrivr-engine-module-jsonl/src/test/kotlin/org/vitrivr/engine/database/jsonl/retrievable/RetrievableWriterTest.kt @@ -0,0 +1,7 @@ +package org.vitrivr.engine.database.jsonl.retrievable + +import org.vitrivr.engine.core.database.retrievable.AbstractRetrievableWriterTest + +class RetrievableWriterTest : AbstractRetrievableWriterTest("test-schema-jsonl.json") { + +} \ No newline at end of file diff --git a/vitrivr-engine-module-jsonl/src/test/resources/test-schema-jsonl.json b/vitrivr-engine-module-jsonl/src/test/resources/test-schema-jsonl.json new file mode 100644 index 00000000..bd000f29 --- /dev/null +++ b/vitrivr-engine-module-jsonl/src/test/resources/test-schema-jsonl.json @@ -0,0 +1,42 @@ +{ + "name": "vitrivr-test", + "connection": { + "database": "JsonlConnectionProvider", + "parameters": { + "root": "jsonl-test" + } + }, + "fields": [ + { + "name": "averagecolor", + "factory": "AverageColor" + }, + { + "name": "file", + "factory": "FileSourceMetadata", + "indexes": [ + { + "attributes": [ + "path" + ], + "type": "FULLTEXT" + }, + { + "attributes": [ + "size" + ], + "type": "SCALAR" + } + ] + }, + { + "name": "time", + "factory": "TemporalMetadata" + }, + { + "name": "video", + "factory": "VideoSourceMetadata" + } + ], + "exporters": [] +} diff --git a/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/Utilities.kt b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/Utilities.kt index 4c620286..8ff0258a 100644 --- a/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/Utilities.kt +++ b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/Utilities.kt @@ -50,6 +50,7 @@ internal fun Type.toSql(): Int = when (this) { Type.Datetime -> JDBCType.DATE Type.String -> JDBCType.VARCHAR Type.Text -> JDBCType.CLOB + Type.UUID -> JDBCType.OTHER is Type.BooleanVector -> JDBCType.ARRAY is Type.DoubleVector -> JDBCType.ARRAY is Type.FloatVector -> JDBCType.ARRAY diff --git a/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/PgDescriptorInitializer.kt b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/PgDescriptorInitializer.kt index 827a39a7..4e7879dd 100644 --- a/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/PgDescriptorInitializer.kt +++ b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/PgDescriptorInitializer.kt @@ -54,6 +54,7 @@ open class PgDescriptorInitializer(final override val field: Sch Type.Float -> statement.append("\"${field.name}\" real, ") Type.Double -> statement.append("\"${field.name}\" double precision, ") Type.Datetime -> statement.append("\"${field.name}\" datetime, ") + Type.UUID -> statement.append("\"${field.name}\" uuid, ") is Type.BooleanVector -> statement.append("\"${field.name}\" bit(${field.type.dimensions}), ") is Type.DoubleVector -> statement.append("\"${field.name}\" vector(${field.type.dimensions}), ") is Type.FloatVector -> statement.append("\"${field.name}\" vector(${field.type.dimensions}), ") diff --git a/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/struct/StructDescriptorReader.kt b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/struct/StructDescriptorReader.kt index d9870e62..7ac1461b 100644 --- a/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/struct/StructDescriptorReader.kt +++ b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/struct/StructDescriptorReader.kt @@ -65,6 +65,7 @@ class StructDescriptorReader(field: Schema.Field<*, StructDescriptor>, connectio Type.Float -> result.getFloat(field.name).let { Value.Float(it) } Type.Double -> result.getDouble(field.name).let { Value.Double(it) } Type.Datetime -> result.getDate(field.name).toInstant().let { Value.DateTime(Date(it.toEpochMilli())) } + Type.UUID -> result.getObject(field.name, UUID::class.java).let { Value.UUIDValue(it) } is Type.BooleanVector -> result.getObject(field.name, PgBitVector::class.java).toBooleanVector() is Type.IntVector -> result.getObject(field.name, PgVector::class.java)?.toIntVector() is Type.LongVector -> result.getObject(field.name, PgVector::class.java)?.toLongVector()