diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..cd17b680 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,50 @@ +name: Java CI with Gradle + +on: + push: + branches: [ main, dev ] + pull_request: + branches: [ main, dev ] + +jobs: + build: + runs-on: ubuntu-latest + + # Setup Cottontail DB and PostgreSQL service container + services: + cottontail: + image: vitrivr/cottontaildb:0.16.7 + ports: + - 1865:1865 + options: -it + pgvector: + image: pgvector/pgvector:pg16 + ports: + - 5432:5432 + env: + POSTGRES_PASSWORD: vitrivr + + # Start actual job. + steps: + - uses: actions/checkout@v4 + - name: Cottontail DB connection test + run: nc -zv 127.0.0.1 1865 + - name: PostgreSQL connection test + run: nc -zv 127.0.0.1 5432 + - name: Set up JDK + uses: actions/setup-java@v4 + with: + java-version: '21' + distribution: 'temurin' + - name: Cache Gradle packages + uses: actions/cache@v4 + with: + path: ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} + restore-keys: ${{ runner.os }}-gradle + - name: Test with gradle ubuntu + if: matrix.os == 'ubuntu-latest' + run: ./gradlew test --info + - name: Test with gradle windows + if: matrix.os == 'windows-latest' + run: ./gradlew test --info \ No newline at end of file diff --git a/README.md b/README.md index b1747458..41b3a18f 100644 --- a/README.md +++ b/README.md @@ -1,1005 +1,78 @@ -# vitrivr engine + + + + +
+
+ + vitrivr log


+ vitrivr-engine logo + +
+
+ +[![GitHub Release](https://img.shields.io/github/release/vitrivr/vitrivr-engine?include_prereleases=&sort=semver&color=blue&style=for-the-badge&label=Release)](https://github.com/vitrivr/vitrivr-engine/releases) +[![License](https://img.shields.io/badge/License-MIT-blue?style=for-the-badge)](#license) +[![issues - vitrivr-engine](https://img.shields.io/github/issues/vitrivr/vitrivr-engine?style=for-the-badge)](https://github.com/vitrivr/vitrivr-engine/issues) [vitrivr](https://vitrivr.org)'s next-generation retrieval engine. -Based on the experiences with its predecessor, [Cineast](https://github.com/vitrivr/cineast), -vitrivr engine's data model, ingestion pipeline and retrieval logic have been reworked. - -## Project Structure - -The project is set up as a multimodule Kotlin project: - -| Module | Description | Maven Dependency | -|-----------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------| -| [`vitrivr-engine-core`](/vitrivr-engine-core) | The core library of the project, which provides basic interfaces & classes. | Yes | -| [`vtirivr-engine-query`](/vitrivr-engine-query) | Query / retrieval related extension to the core library with various retrievers and data manipulation operators. | Yes | -| [`vitrivr-engine-index`](/vitrivr-engine-index) | Indexing / ingestion related extension to the core library with various decoders and segmenters. | Yes | -| [`vitrivr-engine-module-cottontaildb`](/vitrivr-engine-plugin-cottontaildb) | The database driver for the [CottontailDB](https://github.com/vitrivr/cottontaildb) database, used for NNNS and other queries. | Yes | -| [`vitrivr-engine-module-features`](/vitrivr-engine-plugin-features) | Extension that contains specific indexing and retrieval implementations such as fulltext, colour, etc. | Yes | -| [`vitrivr-engine-plugin-m3d`](/vitrivr-engine-plugin-m3d) | Extension related to 3d model indexing and retrieval. Contains various feature modules and capability to process meshes. | Yes | -| [`vitrivr-engine-module-fes`](/vitrivr-engine-module-fes) | Extension that can be used to harnes feature extraction provided by an external ML model server. **Requires local generation of bindings:** `./gradlew :vitrivr-engine-module-fes:generateFESClient` | Yes | -| [`vitrivr-engine-server`](/vitrivr-engine-server) | A [Javalin](https://javalin.io) powered server providing an [OpenApi](https://openapis.org) [documented REST API](vitrivr-engine-server/doc/oas.json) for both, ingestion and querying and a CLI, essentially the runtime of the project | No | - -## Getting Started: Usage - -vitrivr engine is a Kotlin project and hence requires a JDK (e.g. [OpenJDK](https://openjdk.org/)) to properly run. -Furthermore, we use [Gradle](https://gradle.org) in order to facilitate the building and deployment -through the Gradle wrapper. - -In the context of retrieval, often times a distinction of _indexing_ / _ingestion_ (also known as _offline phase_) -and _querying_ / _retrieval_ (also known as _online phase_) is made. -While the former addresses (multimedia) content to be analysed and indexed, i.e. made ready for search, is the latter's purpose to -search within the previously built index. - -### Indexing / Ingestion - -The most essential prerequisite for the ingestion is the existence of multimedia content. -For the sake of this example, let's assume the existence of such multimedia content in the form of image and video files. - -Also, since vitrivr engine is highly configurable, the first few steps involve the creation of a suitable -configuration. - -#### Schema - -vitrivr engine operates on the notion of _schema_, similarly to a database or a collection, -essentially providing, among other things, a namespace. -For this guide, we will have a single schema `sandbox`. - -Create a config file `sandbox-config.json` with one named schema in it: - -```json -{ - "schemas": [{ - "name": "sandbox" - }] -} -``` - -#### Database Connection - -The database is an important component of the system, since it allows for the persistence and lookup of descriptors. -The default database used by `vitrivr-engine` is [CottontailDB](https://github.com/vitrivr/cottontaildb) This guide assumes a running [CottontailDB](https://github.com/vitrivr/cottontaildb) -instance on the same machine on the default port `1865`. - ---- -**NOTE this requires [Cottontail 0.16.5](https://github.com/vitrivr/cottontaildb/releases/tag/0.16.5) or newer** ---- - -We address the database with the [`ConnectionConfig`](/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/ConnectionConfig.kt): - -```json -{ - "database": "CottontailConnectionProvider", - "parameters": { - "Host": "127.0.0.1", - "port": "1865" - } -} -``` - -We add the cottontail connection to the schema's connection property: - -```json -{ - "schemas": [{ - "name": "sandbox", - "connection": { - "database": "CottontailConnectionProvider", - "parameters": { - "Host": "127.0.0.1", - "port": "1865" - } - } - }] -} -``` - -#### Schema Analyser - -The [`Analyser`](/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/metamodel/Analyser.kt) -performs analysis to derive a [`Descriptor`](/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/Descriptor.kt). -In other words, the _analyser_ produces a _descriptor_ which represents the media content analysed. -However, this is only for _indexing_ / _ingestion_. During _querying_ / _retrieval_ time, -the _analyser_ queries the underlying storage layer to perform a query on said _descriptors_. - -#### Schema Field Configuration - -A schema consists of unique-named _fields_, that have to be backed by an [`Analyser`](/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/metamodel/Analyser.kt), -essentially representing a specific [`Descriptor`](/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/Descriptor.kt). -This is configured using the [`FieldConfig`](/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/FieldConfig.kt): - -```json -{ - "name": "uniqueName", - "factory": "FactoryClass", - "parameters":{ - "key": "value" - } -} -``` - -For images (and video frames), it might be worthwhile to use the average colour for representation purposes. -The built-in [`AverageColor`](/vitrivr-engine-plugin-features/src/main/kotlin/org/vitrivr/engine/base/features/averagecolor/AverageColor.kt) -analyser can be facilitated for this endeavour. -To use it, we specifically craft a corresponding _field config_: - -```json -{ - "name": "averagecolor", - "factory": "AverageColor" -} -``` - -There are no additional parameters, unlike, for instance, an [`ExternalAnalyser`](/vitrivr-engine-plugin-features/src/main/kotlin/org/vitrivr/engine/base/features/external/ExternalAnalyser.kt), -which requires the parameter `host` with an endpoint as value. - -For analysers that require a running [Feature Extraction Server](https://github.com/faberf/feature-extraction-server) (FES), the `host` parameter is required. Additionally, the `model` parameter may be used to specify a non-default model which should execute the task. Of course, this requires that the FES has the necessary plugins installed. See the [FES documentation](https://github.com/faberf/feature-extraction-server) for more information. - -- For [`ASR`](/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/ASR.kt) the analyser will perform automatic speech recognition on the audio content. -- For [`OCR`](/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/OCR.kt) the analyser will perform optical character recognition on the image content. -- For [`DenseEmbedding`](/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/DenseEmbedding.kt) the analyser will embed text / images as float vectors. Additionally, the `length` parameter is required to specify the length of the embedding. -- For [`ImageCaption`](/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/ImageCaption.kt) the analyser will generate a caption for the image content. Optionally, a `prompt` parameter can be used to specify a prompt for the caption generation. For example, the prompt could have the form `"Question: What is in the image? Answer:"`. -- For [`ImageClassification`](/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/ImageClassification.kt) the analyser will classify the image content. Additionally, the `classes` parameter is required, which should contain the classes to classify the image into, separated by commas. Optionally, the `top_k` and `threshold` parameters can be used to specify the number of top classes to return and the threshold for the classification. - - -Other fields are for (technical) metadata such as the [`FileSourceMetadata`](/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/metadata/source/file/FileSourceMetadata.kt), -which additionally stores the file's path and size. - -For exif metadata, the [`ExifMetadata`](/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/metadata/source/exif/ExifMetadata.kt) Analyser can be used. For each exif tag that should be included, a parameter with the name "{EXIF_DIRECTORY_NAME}_{TAG_NAME}" must be set to a type. Keys that do not match an exif tag via the aforementioned pattern are interpreted to be custom metadata tags that are stored in the exif UserComment tag in JSON format. Each parameter corresponds to a sub-field. Here is an example with custom "time_zone" metadata: -```json - { - "name": "exif", - "factory": "ExifMetadata", - "parameters": { - "ExifSubIFD_FocalLength": "INT", - "ExifSubIFD_ApertureValue": "FLOAT", - "ExifSubIFD_DateTimeOriginal": "DATETIME", - "ExifSubIFD_MeteringMode": "STRING", - "time_zone": "STRING" - } - } -``` -For extraction, the exif UserComment of images might look like this: -```json -{"time_zone": "Europe/Berlin", "hours_awake": 12} -``` - -Currently, there is no list of available fields and analysers, therefore a quick look into the code -reveals those existent. For basic (metadata), see in [the core module](/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/), -for content-based features, see in [the features' module](/vitrivr-engine-plugin-features/src/main/kotlin/org/vitrivr/engine/base/features/). - -In this guide, we use the _average colour_ and _file source_ fields, which results in the (currently) following -configuration file: - -```json -{ - "schemas": [{ - "name": "sandbox", - "connection": { - "database": "CottontailConnectionProvider", - "parameters": { - "Host": "127.0.0.1", - "port": "1865" - } - }, - "fields": [ - { - "name": "averagecolor", - "factory": "AverageColor" - }, - { - "name": "file", - "factory": "FileSourceMetadata" - } - ] - }] -} -``` - -#### Schema Resolver Configuration - -Some data is stored e.g. on disk during extraction, which later will also be required during query time, -therefore the [`Resolver`](vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/resolver/Resolver.kt) -is configured as the [`ResolverConfig`](vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/schema/ResolverConfig.kt) -on the schema with a unique name. - -The [`ResolverConfig`](vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/schema/ResolverConfig.kt) describes such a configuration: - -```json -{ - "factory": "FactoryClass", - "parameters": { - "key": "value" - } -} -``` - -Specifically, the [`DiskResolver`](/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/resolver/impl/DiskResolver.kt) is implemented and configured as such: - -```json -{ - "factory": "DiskResolver", - "parameters": { - "location": "./thumbnails/vitrivr" - } -} -``` - -Therefore, the _schema_ config is expanded with the _disk resolver_, named `disk`: - -```json -{ - "schemas": [{ - "name": "sandbox", - "connection": { - "database": "CottontailConnectionProvider", - "parameters": { - "Host": "127.0.0.1", - "port": "1865" - } - }, - "fields": [ - { - "name": "averagecolor", - "factory": "AverageColor" - }, - { - "name": "file", - "factory": "FileSourceMetadata" - } - ] - }], - "resolvers": { - "disk": { - "factory": "DiskResolver", - "parameters": { - "location": "./thumbnails/vitrivr" - } - } - } -} -``` - -#### Schema Exporter Configuration - -In the context of images and videos, having thumbnails is desirable, which can be generated -during ingestion with the configuration of an _exporter_. -Generally speaking, an _exporter_ exports an artifact based on the media content. - -The [`ExporterConfig`](/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/ExporterConfig.kt) -describes such a configuration: - -```json -{ - "name": "uniqueName", - "factory": "FactoryClass", - "resolverName": "disk", - "parameters": { - "key": "value" - } -} -``` - -Specifically, the [`ThumbnailExporter`](/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/exporters/ThumbnailExporter.kt), -can be configured as follows, which references a [`DiskResolver`](/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/resolver/impl/DiskResolver.kt) named `disk`, see the previous section. - -```json -{ - "name": "thumbnail", - "factory": "ThumbnailExporter", - "resolverName": "disk", - "parameters": { - "maxSideResolution": "400", - "mimeType": "JPG" - } -} -``` - -Resulting in the following schema config: - -```json -{ - "schemas": [{ - "name": "sandbox", - "connection": { - "database": "CottontailConnectionProvider", - "parameters": { - "Host": "127.0.0.1", - "port": "1865" - } - }, - "fields": [ - { - "name": "averagecolor", - "factory": "AverageColor" - }, - { - "name": "file", - "factory": "FileSourceMetadata" - } - ], - "resolvers": { - "disk": { - "factory": "DiskResolver", - "parameters": { - "location": "./thumbnails/vitrivr" - } - } - }, - "exporters": [ - { - "name": "thumbnail", - "factory": "ThumbnailExporter", - "resolverName": "disk", - "parameters": { - "maxSideResolution": "400", - "mimeType": "JPG" - } - } - ] - }] -} -``` - -#### Extraction Pipeline Configuration - -In order to effectively support a specific _ingestion_ / _indexing_, we have to provide -a reference to the [`IndexConfig`](/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/IndexConfig.kt), -which is configured as a [`PipelineConfig`](/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/PipelineConfig.kt) within the schema config: - -```json -{ - "name": "pipelineName", - "path": "path/to/vbs-config-pipeline.json" -} -``` - -We will create said _index config_ later as `sandbox-pipeline.json`, hence, our schema config -is as follows: - -```json -{ - "schemas": [{ - "name": "sandbox", - "connection": { - "database": "CottontailConnectionProvider", - "parameters": { - "Host": "127.0.0.1", - "port": "1865" - } - }, - "fields": [ - { - "name": "averagecolor", - "factory": "AverageColor" - }, - { - "name": "file", - "factory": "FileSourceMetadata" - } - ], - "exporters": [ - { - "name": "thumbnail", - "factory": "ThumbnailExporter", - "resolver": { - "factory": "DiskResolver", - "parameters": { - "location": "./thumbnails/sandbox" - } - }, - "parameters": { - "maxSideResolution": "400", - "mimeType": "JPG" - } - } - ], - "extractionPipelines": [ - { - "name": "sandboxpipeline", - "path": "./sandbox-pipeline.json" - } - ] - }] -} -``` - -#### Index Pipeline Configuration - -Let's create a new file `sandbox-pipeline.json` right next to the `sandbox-config.json` in the root of the project. -This file will contain the [`IngestionConfig`](/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/ingest/IngestionConfig.kt). - -In order to address (reference) our schema, we reference it in our index config and provide a _context_ as well as an _enumerator_: - -```json -{ - "schema": "sandbox", - "context": { - - }, - "enumerator": { - - } -} -``` - -#### Index Context Configuration - -An [`IngestionContextConfig`](/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/context/IngestionContextConfig.kt) -is used to specify the _context_, additional information to the media data. -Specifically, a [`Resolver`](vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/resolver/Resolver.kt), `disk`, is referenced by its name from the _schema_ configuration. - -```json -{ - "contentFactory": "InMemoryContentFactory", - "resolverName": "disk", - "global": { - "key": "global-value" - }, - "local": { - "operator-name": { - "key": "local-value" - } - } -} -``` - -The [`Context`](vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/context/Context.kt) has two properties: `global` and `local`, which -enable local and global configuration in the form of key-value pairs. -However, `local`ly defined key-value pairs override their `global`ly defined counterpart. -In the above example, for the _operator_ (see further below), the key-value pair with the key `key` is also locally defined, -resulting in the value `local-value` to be used. However, a separate operator named other than `operator-name`, would get the value `global-value` -for the key `key`, as no local value is defined. -We refer to this part of the context, to the _context-parameters_. - -The example index pipeline configuration then, the current context looks as follow: - -```json -{ - "schema": "sandbox", - "context": { - "contentFactory": "InMemoryContentFactory", - "resolverName": "disk" - }, - "enumerator": { - - } -} -``` - -There are these special sections for the context-parameters: - -* `content` - For the local context-parameters for the _content factory_. -#### Index Operators Configuration +[Read The Docs](https://github.com/vitrivr/vitrivr-engine/wiki) -Next up, we declare a list of [operators](vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/operators/Operator.kt) in the form -of [`OperatorConfig`](vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/ingest/operator/OperatorConfig.kt)s. -These _operators_ must have a unique name in the `operators` property of the [`IngestionConfig`](vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/ingest/IngestionConfig.kt): -These names are used in the local context-parameters to configure them. +
-```json -{ - "schema": "sandbox", - "context": { - "contentFactory": "InMemoryContentFactory", - "resolverName": "disk", - "local": { - "enumerator": { - "path": "./sandbox/imgs", - "mediaTypes": "IMAGE;VIDEO", - "depth": "1" - }, - "myoperator1": { - "key1": "a-value", - "key2": "1234" - } - } - }, - "operators": { - "myoperator1": {}, - "myoperator2": {} - } -} -``` +## vitrivr-engine -There are different _types_ of operators: - -* [`Enumerator`](vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/operators/ingest/Enumerator.kt) enumerate and emit `Source`s to ingest from some data source (e.g., the file system). -* [`Decoder`](vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/operators/ingest/Decoder.kt) decode a `Source`s such that the `Content` becomes is available for ingestion (e.g., a video or image decoder). Create and emit `Retrievable`s. -* [`Transformer`](vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/operators/ingest/Segmenter.kt) transform incoming `Retrievable`s and emit 1 to n output `Retrievable`s (e.g., a video segmenter). Result in a 1:1 or 1:n mapping, depending - on the implementation. -* [`Extractor`](vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/operators/ingest/Extractor.kt) extract information from the `Retrievable` and their `Content` and typically append a `Descriptor` before emitting it. -* [`Exporter`](vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/operators/ingest/Exporter.kt) process `Retrievable`s and store external representation thereof (e.g., a preview image). -* [`Aggregator`](vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/operators/ingest/Aggregator.kt) aggregate incoming `Retrievable`s and emit - one [`Retrievable`](vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/retrievable/Retrievable.kt)s, resulting in an n:1 mapping. - -A minimal extraction pipeline requires at least an `Enumerator` and a `Decoder`. - -Notably, `Extractor`s are mostly backed by a schema's field, indicating that the `Descriptor` is being persisted. One can also instantiate `Extractor`s through their -factory, in which case the associated `Descriptor` is transient. `Exporter`s are also referenced by name from the _schema_. - -In the following, we briefly introduce these configurations: - -##### Index Operator Configuration: Enumerator - -The _enumerator_ enumerates the content to index and provides it to the indexing pipeline. -It is described with a [`EnumeratorConfig](/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/ingest/operator/OperatorConfig.kt). - -Requires the property `mediaTypes`, a list of [`MediaType`](/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/source/MediaType.kt)s. - -```json -{ - "type": "ENUMERATOR", - "factory": "FactoryClass", - "mediaTypes": ["IMAGE","VIDEO"] -} -``` - -Currently implemented enumerators are found [in the index module](/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/enumerate), -of which we will use the [`FileSystemEnumerator`](/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/enumerate/FileSystemEnumerator.kt). - -The configuration **requires** the context-parameter `path`, the path to the folder containing multimedia content. -Essentially, for still images, use `IMAGE` and for videos `VIDEO`. -Additional context-parameters are `skip` (how many files should be skipped), `limit` (how many files should at max be enumerated over) -and `depth` (the depth to traverse the file system, `1` stands for current folder only, `2` for sub-folders, `3` for sub-sub-folders, ...). -Let's assume we do have in the root project a folder `sandbox`, with two sub-folders `imgs` and `vids`: - -``` -/sandbox - | - - /imgs - | - - img1.png - | - - img2.png - | - - /vids - | - - vid1.mp4 -``` - -For an image only ingestion, we could set up the configuration as follows (`skip` and `limit` have sensible default values of `0` and `Integer.MAX_VALUE`, respectively): - -**Context**: -```json -{ - "contentFactory": "InMemoryContentFactory", - "resolverName": "disk", - "local": { - "enumerator": { - "path": "./your/media/path", - "depth": "1" - } - } -} -``` - -**Enumerator**: -```json -{ - "type": "ENUMERATOR", - "factory": "FileSystemEnumerator", - "mediaTypes": [ - "IMAGE", "VIDEO" - ] -} -``` - -##### Index Operator Configuration: Decoder - -The [`DecoderConfig`](/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/ingest/operator/OperatorConfig.kt) -describes how the media content is decoded. - -```json -{ - "type": "", - "factory": "DecoderClass" -} -``` - -Available decodes can be found [in the index module](/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/decode). -Since we work with images in this tutorial, we require the [`ImageDecoder`](/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/decode/ImageDecoder.kt): - -```json -{ - "name": "ImageDecoder" -} -``` - -##### Index Operators Configuration: Segmenter - -A [`Segmenter`](vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/operators/ingest/Segmenter.kt) is a 1:n operator, -its [`OperatorConfig`](vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/ingest/operator/OperatorConfig.kt) looks as follows: - -```json -{ - "type": "SEGMENTER", - "factory": "FactoryClass" -} -``` - -The `type` property is mandatory, equally so the `factory`, which has to point to a [`SegmenterFactory`](vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/operators/ingest/SegmenterFactory.kt) implementation. -The context-parameters are optional and implementation dependent. - -See [implementations](vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/segment/) - -##### Index Operators Configuration: Transformer - -A [`Transformer`](vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/operators/ingest/Transformer.kt) is a 1:1 operator, -its [`OperatorConfig`](vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/ingest/operator/OperatorConfig.kt) looks as follows: - -```json -{ - "type": "TRANSFORMER", - "factory": "FactoryClass" -} -``` - -The `type` property is mandatory, equally so the `factory`, which has to point to a [`TransformerFactory`](vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/operators/ingest/TransformerFactory.kt) implementation. -The context-parameters are optional and implementation dependent. - -See [implementations](vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/transform) - -##### Index Operators Configuration: Exporter - -A [`Exporter`](vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/operators/ingest/Exporter.kt) is a 1:1 operator, -its [`OperatorConfig`](vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/ingest/operator/OperatorConfig.kt) looks as follows: - -```json -{ - "type": "EXPORTER", - "exporterName": "name-from-schema" -} -``` - -The `type` property is mandatory, equally so the `exporterName`, which has to point to an `Exporter` defined on the _schema_. -The context-parameters are optional and implementation dependent and override those present in the _schema_ configuration. - -See [implementations](vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/exporters) - -##### Index Operators Configuration: Extractor - -A [`Extractor`](vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/operators/ingest/Extractor.kt) is a 1:1 operator, -its [`OperatorConfig`](vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/ingest/operator/OperatorConfig.kt) looks as follows: - -```json -{ - "type": "EXTRACTOR", - "fieldName": "name-from-schema" -} -``` - -The `type` property is mandatory, equally so the `fieldName`, which has to point to a _field_ as defined on the _schema_. - -See [implementations](vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/base/features/) - -##### Index Operators Configuration: Aggregator - -A [`Aggregator`](vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/operators/ingest/Aggregator.kt) is a 1:n operator, -its [`OperatorConfig`](vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/ingest/operator/OperatorConfig.kt) looks as follows: - -```json -{ - "type": "AGGREGATOR", - "factory": "FactoryClass" -} -``` - -The `type` property is mandatory, equally so the `factory`, which has to point to a [`AggregatorFactory`](vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/operators/ingest/AggregatorFactory.kt) implementation. -The context-parameters are optional and implementation dependent. - -See [implementations](vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators) - -#### Index Operations Configuration: The Pipeline - -So far, we only have _declared_ the operators, with the `operations` property, we define the ingestion pipeline as a tree in the form of -[`OperationsConfig`](vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/ingest/operation/OperationsConfig.kt): - -```json -{ - "schema": "sandbox", - "context": { - "contentFactory": "InMemoryContentFactory", - "resolverName": "disk", - "local": { - "enumerator": { - "path": "./sandbox/imgs", - "depth": "1" - }, - "myoperator": { - "key1": "a-avlue", - "key2": "1234" - } - } - }, - "operators": { - "myoperator": {}, - "myoperator1": {}, - "myoperator2": {} - }, - "operations": { - "myOperation": { - "operator": "myoperator" - }, - "myOperation1": { - "operator": "myoperator1", "inputs": ["myOperation"] - }, - "myOperation2": { - "operator": "myoperator2", "inputs": ["myOperation"] - } - }, - "output": ["myOperation2"] -} -``` - -Specifically, the `operator` property must point to a previously declared _operator_ and -the entries in the `inputs` property must point to an _operation_ with that name. -The `output` property of the ingest config then specifies the _operation_ whose results are persisted. - -Currently, there are the following rules to build such a pipeline: - -**Pipeline Rules:** - -1. The first _operation_ **must** be a `ENUMERATOR` -2. Following an `ENUMERATOR`, there **must** come a `DECODER` -3. Following a `DECODER`, there **must** either be a `TRANSFORMER` or `SEGMENTER` -4. `TRANSFORMER`s and `SEGMENTER`s can be daisy-chained -5. A `SEGMENTER` must be followed by one or more `AGGREGATOR`s, multiple `AGGREGATORS` results in branching. -6. An `AGGREGATOR` must be followed by either a `EXTRACTOR` or `EXPORTER` -7. `EXPORTER`s and `EXTRACTOR`s can be daisy-chained -8. The end or the ends, in case of branching, must be of type `EXPORTER` or `EXTRACTOR`. - -Notably, currently multiple `ENUMERATORS` are treated as separate trees, since merging is not yet supported. - -One example, based on the _schema_ further above (without branching), might look as follows: - -```json -{ - "schema": "sandbox", - "context": { - "contentFactory": "InMemoryContentFactory", - "resolverName": "disk", - "local": { - "enumerator": { - "path": "./sandbox/imgs", - "depth": "1" - }, - "thumbs": { - "maxSideResolution": "350", - "mimeType": "JPG" - } - } - }, - "operators": { - "fsenumerator": { - "type": "ENUMERATOR", - "factory": "FileSystemEnumerator", - "mediaTypes": ["IMAGE"] - }, - "decoder": { - "type": "DECODER", - "factory": "ImageDecoder" - }, - "pass": { - "type": "SEGMENTER", - "factory": "PassThroughSegmenter" - }, - "allContent": { - "type": "AGGREGATOR", - "factory": "AllContentAggregator" - }, - "avgColor": { - "type": "EXTRACTOR", - "fieldName": "averagecolor" - }, - "thumbs": { - "type": "EXPORTER", - "exporterName": "thumbnail" - }, - "fileMeta": { - "type": "EXTRACTOR", - "fieldName": "file" - } - }, - "operations": { - "stage2": {"operator": "pass", "inputs": ["stage1"]}, - "stage0": {"operator": "fsenumerator"}, - "stage1": {"operator": "decoder", "inputs": ["stage0"]}, - "stage3": {"operator": "allContent", "inputs": ["stage2"]}, - "stage4": {"operator": "avgColor", "inputs": ["stage3"]}, - "stage5": {"operator": "thumbs", "inputs": ["stage4"]}, - "stage6": {"operator": "fileMeta", "inputs": ["stage5"]} - }, - "output": ["stage6"] -} -``` - -Here, the linear order of the pipeline's operator is: `fsenumerator` -> `decoder` -> `pass` -> `allContent` -> `avgColor` -> `thumbs` -> `fileMeta`. -Note that there are context-parameters defined for the `thumbs` exporter. -Also, note that the `output` property is set up, such that the `stage6` operation is persisted. - -#### Branching and Merging - -By defining the operations accordingly, there are two thing that can happen implicitly. - -**Branching**: If an `operations` is used as input for multiple other `operations`, this results in a branching. This is handled automatically by wrapping the associated `Operator` in a `BroadcastOperator`. - -**Merging**: If an `operation` has multiple inputs, this results in a merging, which combines multiple flows of `Retrievable`s into a single flow. The merging strategy (`MergeType`) must be specified explicitly in the `operation`. - -Currently, `vitrivr-engine` supports three type of merging strategies: - -| MergeType | Description | -|-----------|----------------------------------------------------------------------------------------------------------------------| -| `MERGE` | Merges the `Retrievable`s from the input operations in order the arrive. No deduplication and ordering is performed. | -| `COMBINE` | Merges `Retrievable`s from the input operations and emits a `Retrievable`, once it was received on every input. | -| `CONCAT` | Collects `Retrievable`s from the incoming flows in order of occurence, i.e., operation 1, then operation 2 etc. | - -#### Complete Sandbox Configuration - -After following above's guide on how to build your _schema_ config and your _index pipeline_ config, -the files should be similar as follows. - -The **schema** config: - -```json -{ - "schemas": [{ - "name": "sandbox", - "connection": { - "database": "CottontailConnectionProvider", - "parameters": { - "Host": "127.0.0.1", - "port": "1865" - } - }, - "fields": [ - { - "name": "averagecolor", - "factory": "AverageColor" - }, - { - "name": "file", - "factory": "FileSourceMetadata" - } - ], - "exporters": [ - { - "name": "thumbnail", - "factory": "ThumbnailExporter", - "resolver": { - "factory": "DiskResolver", - "parameters": { - "location": "./thumbnails/sandbox" - } - }, - "parameters": { - "maxSideResolution": "400", - "mimeType": "JPG" - } - } - ], - "extractionPipelines": [ - { - "name": "sandboxpipeline", - "path": "./sandbox-pipeline.json" - } - ] - }] -} -``` - -The **pipeline** config: - -```json -{ - "schema": "sandbox", - "context": { - "contentFactory": "InMemoryContentFactory", - "resolverName": "disk", - "local": { - "fsenumerator": { - "path": "./sandbox/imgs", - "depth": "1" - }, - "thumbs": { - "path": "./sandbox/thumbnails", - "maxSideResolution": "350", - "mimeType": "JPG" - } - } - }, - "operators": { - "fsenumerator": { - "type": "ENUMERATOR", - "factory": "FileSystemEnumerator", - "mediaTypes": ["IMAGE","VIDEO"] - }, - "decoder": { - "type": "DECODER", - "factory": "ImageDecoder" - }, - "pass": { - "type": "SEGMENTER", - "factory": "PassThroughSegmenter" - }, - "allContent": { - "type": "AGGREGATOR", - "factory": "AllContentAggregator" - }, - "avgColor": { - "type": "EXTRACTOR", - "fieldName": "averagecolor" - }, - "thumbs": { - "type": "EXPORTER", - "exporterName": "thumbnail" - }, - "fileMeta": { - "type": "EXTRACTOR", - "fieldName": "file" - } - }, - "operations": { - "stage2": {"operator": "pass", "inputs": ["stage1"]}, - "stage0": {"operator": "fsenumerator"}, - "stage1": {"operator": "decoder", "inputs": ["stage0"]}, - "stage3": {"operator": "allContent", "inputs": ["stage2"]}, - "stage4": {"operator": "avgColor", "inputs": ["stage3"]}, - "stage5": {"operator": "thumbs", "inputs": ["stage4"]}, - "stage6": {"operator": "fileMeta", "inputs": ["stage5"]} - }, - "output" : ["stage6"] -} -``` - ---- - -#### Starting the indexing pipeline - -To start the actual pipeline, we start [the server module](/vitrivr-engine-server)'s [`Main`](vitrivr-engine-server/src/main/kotlin/org/vitrivr/engine/server/Main.kt) -with the path to the schema configuration as argument. +vitrivr-engine is [vitrivr](https://vitrivr.org)'s next generation retrieval engine with a flexible, modular architecture. +Based on the experiences with its predecessor, [Cineast](https://github.com/vitrivr/cineast), +vitrivr engine's data model, ingestion pipeline and retrieval logic have been reworked from the ground. +Essentially, vitrivr-engine enables the analysis (i.e. ingestion) and querying (i.e. retrieval ) of +multimedia data. -For this to work you either build the stack or you use an IDE. +## Built With -1. Then, when the server is running, we have to first initialise the database (since our schema is named `sandbox`): +* [Kotlin](https://kotlinlang.org) for the JVM, e.g. [OpenJDK](https://openjdk.org/) +* [OpenApi](https://www.openapis.org/) +* [CottontailDB](https://github.com/vitrivr/CottontailDB) +* ... and more ... -``` -sandbox init -``` +## Getting Started -2. The extraction is started via the CLI by calling: -``` -sandbox extract -c sandbox-pipeline.json -``` +See [Getting Started](https://github.com/vitrivr/vitrivr-engine/wiki/Getting-Started) -Which should result in logging messages that confirm the usage of our ThumbnailExporter (including its parameters) and the message: -``` -Started extraction job with UUID -``` +## Usage -3. The server (by default) provides an [OpenAPI swagger ui](http://localhost:7070/swagger-ui) with which the job status can be queried. -The same can be achieved by this cURL command, where `` is the UUID printed above (and again, we have named our schema `sandbox`, hence the sandbox path: +See [Example](https://github.com/vitrivr/vitrivr-engine/wiki/Example) -```bash -curl -X 'GET' \ - 'http://localhost:7070/api/sandbox/index/' \ - -H 'accept: application/json' -``` +## Project Structure -### Querying / Retrieval +The project is set up as a multi-module Kotlin project: -## Getting Started: Development +| Module | Description | Maven Dependency | +|-----------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------| +| [`vitrivr-engine-core`](https://github.com/vitrivr/vitrivr-engine//vitrivr-engine-core) | The core library of the project, which provides basic interfaces & classes. | Yes | +| [`vtirivr-engine-query`](https://github.com/vitrivr/vitrivr-engine//vitrivr-engine-query) | Query / retrieval related extension to the core library with various retrievers and data manipulation operators. | Yes | +| [`vitrivr-engine-index`](https://github.com/vitrivr/vitrivr-engine//vitrivr-engine-index) | Indexing / ingestion related extension to the core library with various decoders and segmenters. | Yes | +| [`vitrivr-engine-module-cottontaildb`](https://github.com/vitrivr/vitrivr-engine//vitrivr-engine-plugin-cottontaildb) | The database driver for the [CottontailDB](https://github.com/vitrivr/cottontaildb) database, used for NNNS and other queries. | Yes | +| [`vitrivr-engine-module-features`](https://github.com/vitrivr/vitrivr-engine//vitrivr-engine-plugin-features) | Extension that contains specific indexing and retrieval implementations such as fulltext, colour, etc. | Yes | +| [`vitrivr-engine-module-m3d`](https://github.com/vitrivr/vitrivr-engine//vitrivr-engine-module-m3d) | Extension related to 3d model indexing and retrieval. Contains various feature modules and capability to process meshes. | Yes | +| [`vitrivr-engine-module-fes`](https://github.com/vitrivr/vitrivr-engine//vitrivr-engine-module-fes) | Extension that can be used to harnes feature extraction provided by an external ML model server. **Requires local generation of bindings:** `./gradlew :vitrivr-engine-module-fes:generateFESClient` | Yes | +| [`vitrivr-engine-server`](https://github.com/vitrivr/vitrivr-engine//vitrivr-engine-server) | A [Javalin](https://javalin.io) powered server providing an [OpenApi](https://openapis.org) [documented REST API](vitrivr-engine-server/doc/oas.json) for both, ingestion and querying and a CLI, essentially the runtime of the project | No | -This is a Gradle-powered Kotlin project, we assume prerequisites are handled accordingly. +## Contributing -1. Generate the OpenApi client code by executing the (top-level) `generateOpenApi` gradle task -2. Start developing +We welcome contributors. Please fork the repo and open a pull-request with your work. +A good starting point are the 'good first issue' issues. -If you develop another module (plugin), please keep in mind that the providers and factories are -exposed in `/resources/META-INF/services/` +## Contributors -``` -./gradlew openApiGenerate -``` +* @ppanopticon +* @lucaro +* @sauterl +* @net-csscience-raphel +* @rahelarnold98 +* @faberf +## Citation +See [Citation](https://github.com/vitrivr/vitrivr-engine/wiki/Home#Citation) diff --git a/build.gradle b/build.gradle index 1e168a18..e1d10211 100644 --- a/build.gradle +++ b/build.gradle @@ -47,9 +47,10 @@ subprojects { } /* Common plugins. */ - apply plugin: 'java' apply plugin: 'idea' apply plugin: 'kotlin' + apply plugin: 'java' + apply plugin: 'java-test-fixtures' /* Java Version */ sourceCompatibility = JavaVersion.VERSION_21 diff --git a/config-pipelines/image-pipeline.json b/config-pipelines/image-pipeline.json deleted file mode 100644 index ce6ae026..00000000 --- a/config-pipelines/image-pipeline.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "schema": "vitrivr", - "context": { - "contentFactory": "InMemoryContentFactory", - "resolverFactory": "DiskResolver" - }, - "enumerator": { - "name": "ApiEnumerator", - "parameters": { - "depth": "5", - "mediaTypes": "IMAGE" - }, - "next": { - "name": "VideoDecoder", - "nextTransformer": { - "name": "PassthroughTransformer", - "nextSegmenter": { - "name": "FixedDurationSegmenter", - "parameters": { - "duration": "1", - "lookAheadTime": "1" - }, - "aggregators": [ - { - "name": "AverageImageContentAggregator", - "nextExtractor": { - "fieldName": "averagecolor", - "nextExporter": { - "name": "ThumbnailExporter", - "exporterName": "thumbnail", - - "nextExtractor": { - "fieldName": "file", - "nextExtractor": { - "fieldName": "time", - "nextExtractor": { - "fieldName": "clip", - "nextExtractor": { - "fieldName": "dino" - } - } - } - } - } - } - } - ] - } - } - } - } -} \ No newline at end of file diff --git a/config-pipelines/video-pipeline.json b/config-pipelines/video-pipeline.json deleted file mode 100644 index 2a2ab108..00000000 --- a/config-pipelines/video-pipeline.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "schema": "vitrivr", - "context": { - "contentFactory": "InMemoryContentFactory", - "resolverFactory": "DiskResolver" - }, - "enumerator": { - "name": "ApiEnumerator", - "parameters": { - "depth": "5", - "mediaTypes": "VIDEO" - }, - "next": { - "name": "VideoDecoder", - "nextTransformer": { - "name": "PassthroughTransformer", - "nextSegmenter": { - "name": "FixedDurationSegmenter", - "parameters": { - "duration": "1", - "lookAheadTime": "1" - }, - "aggregators": [ - { - "name": "AverageImageContentAggregator", - "nextExtractor": { - "fieldName": "averagecolor", - "nextExporter": { - "name": "ThumbnailExporter", - "exporterName": "thumbnail", - "nextExtractor": { - "fieldName": "file", - "nextExtractor": { - "fieldName": "time", - "nextExtractor": { - "fieldName": "clip", - "nextExtractor": { - "fieldName": "dino" - } - } - } - } - } - } - } - ] - } - } - } - } -} \ No newline at end of file diff --git a/config-schema.json b/config-schema.json index ce812ba0..fc7f88b9 100755 --- a/config-schema.json +++ b/config-schema.json @@ -5,7 +5,7 @@ "connection": { "database": "CottontailConnectionProvider", "parameters": { - "Host": "127.0.0.1", + "host": "127.0.0.1", "port": "1865" } }, diff --git a/gradle.properties b/gradle.properties index 0706c158..4b89fd8f 100644 --- a/gradle.properties +++ b/gradle.properties @@ -7,6 +7,7 @@ version_javacv=1.5.9 version_javalin=6.1.3 version_javalinopenapi=6.1.3 version_javalinssl=6.1.3 +version_jdbc_postgres=42.7.3 version_jline=3.23.0 version_junit=5.10.1 version_junit_platform=1.10.1 diff --git a/images/vengine-256.png b/images/vengine-256.png new file mode 100755 index 00000000..a59b8cdd Binary files /dev/null and b/images/vengine-256.png differ diff --git a/images/vengine-512.png b/images/vengine-512.png new file mode 100755 index 00000000..5ef04d62 Binary files /dev/null and b/images/vengine-512.png differ diff --git a/images/vitrivr_256.png b/images/vitrivr_256.png new file mode 100755 index 00000000..ce0efabd Binary files /dev/null and b/images/vitrivr_256.png differ diff --git a/images/vitrivr_512.png b/images/vitrivr_512.png new file mode 100755 index 00000000..f3fb5c9c Binary files /dev/null and b/images/vitrivr_512.png differ diff --git a/settings.gradle b/settings.gradle index db29d6cb..0ad7ee17 100644 --- a/settings.gradle +++ b/settings.gradle @@ -8,8 +8,8 @@ include 'vitrivr-engine-index' include 'vitrivr-engine-query' include 'vitrivr-engine-server' include 'vitrivr-engine-module-cottontaildb' +include 'vitrivr-engine-module-pgvector' include 'vitrivr-engine-module-features' include 'vitrivr-engine-module-m3d' include 'vitrivr-engine-module-fes' include 'vitrivr-engine-module-statistics' - diff --git a/vitrivr-engine-core/build.gradle b/vitrivr-engine-core/build.gradle index 40db6ce5..8ad619c8 100755 --- a/vitrivr-engine-core/build.gradle +++ b/vitrivr-engine-core/build.gradle @@ -7,10 +7,15 @@ plugins { dependencies { /** JOML dependencies for 3D mesh support. */ implementation group: 'org.joml', name: 'joml', version: version_joml + /** dependencies for exif metadata extraction. */ implementation 'com.drewnoakes:metadata-extractor:2.19.0' implementation 'com.google.code.gson:gson:2.8.9' implementation group: 'io.javalin.community.openapi', name: 'javalin-openapi-plugin', version: version_javalinopenapi + + /* Test Fixtures from Cottontail DB core. .*/ + testFixturesImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-api', version: version_junit + testFixturesImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-params', version: version_junit } /* Publication of vitrivr engine core to Maven Central. */ diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/schema/FieldConfig.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/schema/FieldConfig.kt index d3c2b4ce..f20245e4 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/schema/FieldConfig.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/schema/FieldConfig.kt @@ -11,6 +11,5 @@ import org.vitrivr.engine.core.model.metamodel.Schema * @author Ralph Gasser * @version 1.0.0 */ - @Serializable -data class FieldConfig(val name: String, val factory: String, val parameters: Map = emptyMap()) +data class FieldConfig(val name: String, val factory: String, val parameters: Map = emptyMap(), val indexes: List = emptyList()) diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/schema/IndexConfig.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/schema/IndexConfig.kt new file mode 100644 index 00000000..8eef81e3 --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/schema/IndexConfig.kt @@ -0,0 +1,20 @@ +package org.vitrivr.engine.core.config.schema + +import kotlinx.serialization.Serializable +import org.vitrivr.engine.core.model.descriptor.AttributeName +import org.vitrivr.engine.core.model.metamodel.Schema + +/** + * A serializable configuration object to configure indexes for a [Schema.Field] within a [Schema]. + * + * @see [Schema.Field] + * + * @author Ralph Gasser + * @version 1.0.0 + */ +@Serializable +data class IndexConfig(val attributes: List, val type: IndexType, val parameters: Map = emptyMap()) { + init { + require(attributes.isNotEmpty()) { "Cannot define index on empty list of attributes." } + } +} \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/schema/IndexType.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/schema/IndexType.kt new file mode 100644 index 00000000..ea6aabf2 --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/schema/IndexType.kt @@ -0,0 +1,15 @@ +package org.vitrivr.engine.core.config.schema + +/** + * The type of index that can be created on a [Schema.Field]. + */ +enum class IndexType { + /** A B-tree based index. Well-suited for point-lookups and Boolean search on scalar values. */ + SCALAR, + + /** A fulltext index. Well-suited for fulltext-queries. */ + FULLTEXT, + + /** Index suited for NNS. */ + NNS +} \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/schema/SchemaConfig.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/schema/SchemaConfig.kt index 77a42d55..1350706c 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/schema/SchemaConfig.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/config/schema/SchemaConfig.kt @@ -1,9 +1,12 @@ package org.vitrivr.engine.core.config.schema import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.operators.general.Exporter import org.vitrivr.engine.core.resolver.Resolver +import java.nio.file.Files +import java.nio.file.Paths /** @@ -43,4 +46,22 @@ data class SchemaConfig( * List of [PipelineConfig]s that are part of this [SchemaConfig]. */ val extractionPipelines: List = emptyList() -) +) { + + companion object { + + /** + * Tries to load a [SchemaConfig] from the resources. + * + * @param resourcePath Path to the resource. + * @return [SchemaConfig] + */ + fun loadFromResource(resourcePath: String): SchemaConfig { + val json = Json { ignoreUnknownKeys = true } // Configure Json to ignore unknown keys + val uri = this::class.java.classLoader.resources(resourcePath).findFirst().orElseThrow { IllegalArgumentException("Resource '$resourcePath' not found!") }.toURI() + val path = Paths.get(uri) + val jsonString = Files.readString(path) + return json.decodeFromString(jsonString) + } + } +} diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/AbstractConnectionProvider.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/AbstractConnectionProvider.kt new file mode 100644 index 00000000..a20582a2 --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/AbstractConnectionProvider.kt @@ -0,0 +1,50 @@ +package org.vitrivr.engine.core.database + +import org.vitrivr.engine.core.database.descriptor.DescriptorProvider +import org.vitrivr.engine.core.model.descriptor.Descriptor +import kotlin.reflect.KClass + +/** + * Abstract implementation of the [ConnectionProvider] interface, which provides basic facilities to register [DescriptorProvider]s. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +abstract class AbstractConnectionProvider: ConnectionProvider { + /** A map of registered [DescriptorProvider]. */ + protected val registered: MutableMap, DescriptorProvider<*>> = HashMap() + + + init { + this.initialize() + } + + /** + * This method is called during initialization of the [AbstractConnectionProvider] and can be used to register [DescriptorProvider]s. + */ + abstract fun initialize() + + /** + * Registers an [DescriptorProvider] for a particular [KClass] of [Descriptor] with this [Connection]. + * + * This method is an extension point to add support for new [Descriptor]s to a pre-existing database driver. + * + * @param descriptorClass The [KClass] of the [Descriptor] to register [DescriptorProvider] for. + * @param provider The [DescriptorProvider] to register. + */ + override fun register(descriptorClass: KClass, provider: DescriptorProvider<*>) { + this.registered[descriptorClass] = provider + } + + /** + * Obtains an [DescriptorProvider] for a particular [KClass] of [Descriptor], that has been registered with this [ConnectionProvider]. + * + * @param descriptorClass The [KClass] of the [Descriptor] to lookup [DescriptorProvider] for. + * @return The registered [DescriptorProvider] . + */ + @Suppress("UNCHECKED_CAST") + override fun obtain(descriptorClass: KClass): DescriptorProvider { + val provider = this.registered[descriptorClass] ?: throw IllegalStateException("No DescriptorProvider registered for $descriptorClass.") + return provider as DescriptorProvider + } +} \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/Connection.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/Connection.kt index 0ee9c29d..e1e039d9 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/Connection.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/Connection.kt @@ -25,6 +25,13 @@ sealed interface Connection: Closeable { /** The name of the [Schema] managed by this [Connection]. */ val schemaName: String + /** + * Executes the provided action within a transaction (if supported by the database). + * + * @param action The action to execute within the transaction. + */ + fun withTransaction(action: (Unit) -> T): T + /** * Initializes the database layer with the [Schema] used by this [Connection]. */ diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/Initializer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/Initializer.kt index 553ebe8c..e7651418 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/Initializer.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/Initializer.kt @@ -1,6 +1,7 @@ package org.vitrivr.engine.core.database import org.vitrivr.engine.core.model.Persistable +import org.vitrivr.engine.core.model.query.basics.Distance /** * The [Initializer] is part of the data persistence layer of vitrivr and can be used to encapsulate DDL operations @@ -8,14 +9,28 @@ import org.vitrivr.engine.core.model.Persistable * * @author Ralph Gasser * @author Luca Rossetto - * @version 1.0.0 + * @version 1.1.0 */ interface Initializer { + + companion object { + /** The column name used to specify type of index. This is database specific.*/ + const val INDEX_TYPE_PARAMETER_NAME = "type" + + /** The parameter name used to specify the distance. Used upon index creation. References [Distance].*/ + const val DISTANCE_PARAMETER_NAME = "distance" + } + /** * Initializes the (persistent) data structures required by the [Persistable] of type [T]. */ fun initialize() + /** + * De-initializes the (persistent) data structures required by the [Persistable] of type [T]. + */ + fun deinitialize() + /** * Returns true if the structures created by this [Initializer] have already been created and setup. * diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/Reader.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/Reader.kt index abba9526..4fe3c5a0 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/Reader.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/Reader.kt @@ -13,6 +13,9 @@ import java.util.* * @version 1.1.0 */ interface Reader { + /** The [Connection] used by this [Writer]. */ + val connection: Connection + /** * Returns a [Sequence] of all [Persistable] accessible by this [Reader]. * diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/Writer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/Writer.kt index cf915455..35c8a1c0 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/Writer.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/Writer.kt @@ -15,6 +15,9 @@ import java.util.* */ interface Writer { + /** The [Connection] used by this [Writer]. */ + val connection: Connection + /** * Adds (and typically persists) a single [Persistable] of type [T] through this [Writer]. * diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/descriptor/DescriptorReader.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/descriptor/DescriptorReader.kt index 53837a0b..5df0973d 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/descriptor/DescriptorReader.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/descriptor/DescriptorReader.kt @@ -17,21 +17,13 @@ import org.vitrivr.engine.core.model.retrievable.Retrieved * * @author Luca Rossetto * @author Ralph Gasser - * @version 2.0.0 + * @version 2.0.1 */ interface DescriptorReader : Reader { /** The [Analyser] this [DescriptorReader] belongs to. */ val field: Schema.Field<*,D> - /** - * Returns the [Descriptor] of type [D] that corresponds to the provided [DescriptorId]. - * - * @param descriptorId The [DescriptorId] to return. - * @return [Descriptor] of type [D] or null - */ - operator fun get(descriptorId: DescriptorId): D? - /** * Checks if a [Descriptor] of type [D] for provided [DescriptorId] exists. * @@ -41,12 +33,12 @@ interface DescriptorReader : Reader { fun exists(descriptorId: DescriptorId): Boolean /** - * Returns the [Descriptor]s of type [D] that belong to the provided [RetrievableId]. + * Returns the [Descriptor] of type [D] that corresponds to the provided [DescriptorId]. * - * @param retrievableId The [RetrievableId] to search for. - * @return [Sequence] of [Descriptor] of type [D] + * @param descriptorId The [DescriptorId] to return. + * @return [Descriptor] of type [D] or null */ - fun getFor(retrievableId: RetrievableId): Sequence + operator fun get(descriptorId: DescriptorId): D? /** * Returns a [Sequence] of all [Descriptor] whose [DescriptorId] is contained in the provided [Iterable]. @@ -56,13 +48,21 @@ interface DescriptorReader : Reader { */ fun getAll(descriptorIds: Iterable): Sequence + /** + * Returns the [Descriptor]s of type [D] that belong to the provided [RetrievableId]. + * + * @param retrievableId The [RetrievableId] to search for. + * @return [Sequence] of [Descriptor] of type [D] + */ + fun getForRetrievable(retrievableId: RetrievableId): Sequence + /** * Returns a [Sequence] of all [Descriptor] whose [RetrievableId] is contained in the provided [Iterable]. * * @param retrievableIds A [Iterable] of [RetrievableId]s to return [Descriptor]s for * @return [Sequence] of [Descriptor] of type [D] */ - fun getAllFor(retrievableIds: Iterable): Sequence + fun getAllForRetrievable(retrievableIds: Iterable): Sequence /** * Returns a [Sequence] of all [Descriptor]s [D]s that match the given [Query]. diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/descriptor/NoDescriptorInitializer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/descriptor/NoDescriptorInitializer.kt index 6ca7a468..773d1798 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/descriptor/NoDescriptorInitializer.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/descriptor/NoDescriptorInitializer.kt @@ -12,10 +12,15 @@ import org.vitrivr.engine.core.model.metamodel.Schema * @version 1.0.0 */ class NoDescriptorInitializer(override val field: Schema.Field<*,D>) : DescriptorInitializer { - override fun initialize() { /* No op. */ + override fun initialize() { + /* No op. */ } + override fun deinitialize() { + /* No op. */ + } override fun isInitialized(): Boolean = false - override fun truncate() { /* No op. */ + override fun truncate() { + /* No op. */ } } \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/retrievable/NoRetrievableInitializer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/retrievable/NoRetrievableInitializer.kt index dafae4a6..a1e3be93 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/retrievable/NoRetrievableInitializer.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/retrievable/NoRetrievableInitializer.kt @@ -9,10 +9,17 @@ package org.vitrivr.engine.core.database.retrievable * @version 1.0.0 */ class NoRetrievableInitializer : RetrievableInitializer { - override fun initialize() { /* No op. */ + override fun initialize() { + /* No op. */ } - override fun isInitialized(): Boolean = false - override fun truncate() { /* No op. */ + override fun deinitialize() { + /* No op. */ + } + + override fun truncate() { + /* No op. */ } + + override fun isInitialized(): Boolean = false } \ No newline at end of file diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/averagecolor/AverageColor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColor.kt similarity index 81% rename from vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/averagecolor/AverageColor.kt rename to vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColor.kt index 2510922b..9d9769e3 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/averagecolor/AverageColor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColor.kt @@ -1,7 +1,5 @@ -package org.vitrivr.engine.module.features.feature.averagecolor +package org.vitrivr.engine.core.features.averagecolor -import io.github.oshai.kotlinlogging.KLogger -import io.github.oshai.kotlinlogging.KotlinLogging import org.vitrivr.engine.core.context.IndexContext import org.vitrivr.engine.core.context.QueryContext import org.vitrivr.engine.core.model.color.MutableRGBFloatColorContainer @@ -25,23 +23,42 @@ import java.util.* /** * Implementation of the [AverageColor] [Analyser], which derives the average color from an [ImageContent] as [FloatVectorDescriptor]. * + * This [Analyser] has little practical relevance these days but acts as a simple example for how to create a custom [Analyser] that uses vectors. + * Furthermore, it can be used to implement test cases. + * * @author Ralph Gasser * @version 1.0.0 */ class AverageColor : Analyser { - private val logger: KLogger = KotlinLogging.logger {} - override val contentClasses = setOf(ImageContent::class) override val descriptorClass = FloatVectorDescriptor::class + companion object { + /** + * Performs the [AverageColor] analysis on the provided [List] of [ImageContent] elements. + * + * @param content The [List] of [ImageContent] elements. + * @return [List] of [FloatVectorDescriptor]s. + */ + fun analyse(content: Collection): List = content.map { + val color = MutableRGBFloatColorContainer() + val rgb = it.content.getRGBArray() + rgb.forEach { c -> color += RGBByteColorContainer(c) } + + /* Generate descriptor. */ + val averageColor = RGBFloatColorContainer(color.red / rgb.size, color.green / rgb.size, color.blue / rgb.size) + FloatVectorDescriptor(UUID.randomUUID(), null, averageColor.toVector()) + } + } + /** * Generates a prototypical [FloatVectorDescriptor] for this [AverageColor]. * * @param field [Schema.Field] to create the prototype for. * @return [FloatVectorDescriptor] */ - override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), listOf(Value.Float(0.0f), Value.Float(0.0f), Value.Float(0.0f))) + override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.FloatVector(3)) /** * Generates and returns a new [AverageColorExtractor] instance for this [AverageColor]. @@ -78,9 +95,9 @@ class AverageColor : Analyser { */ override fun newRetrieverForQuery(field: Schema.Field, query: Query, context: QueryContext): AverageColorRetriever { require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } - require(query is ProximityQuery<*> && query.value.first() is Value.Float) { "The query is not a ProximityQuery." } + require(query is ProximityQuery<*> && query.value is Value.FloatVector) { "The query is not a ProximityQuery." } @Suppress("UNCHECKED_CAST") - return AverageColorRetriever(field, query as ProximityQuery) + return AverageColorRetriever(field, query as ProximityQuery) } /** @@ -114,22 +131,5 @@ class AverageColor : Analyser { * @param context The [QueryContext] to use with the [Retriever] */ override fun newRetrieverForContent(field: Schema.Field, content: Collection, context: QueryContext): AverageColorRetriever = - this.newRetrieverForDescriptors(field, this.analyse(content), context) - - /** - * Performs the [AverageColor] analysis on the provided [List] of [ImageContent] elements. - * - * @param content The [List] of [ImageContent] elements. - * @return [List] of [FloatVectorDescriptor]s. - */ - fun analyse(content: Collection): List = content.map { - logger.trace{"Analysing"} - val color = MutableRGBFloatColorContainer() - val rgb = it.content.getRGBArray() - rgb.forEach { c -> color += RGBByteColorContainer.fromRGB(c) } - - /* Generate descriptor. */ - val averageColor = RGBFloatColorContainer(color.red / rgb.size, color.green / rgb.size, color.blue / rgb.size) - FloatVectorDescriptor(UUID.randomUUID(), null, averageColor.toValueList()) - } + this.newRetrieverForDescriptors(field, analyse(content), context) } diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/averagecolor/AverageColorExtractor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColorExtractor.kt similarity index 90% rename from vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/averagecolor/AverageColorExtractor.kt rename to vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColorExtractor.kt index 4bf0afcc..82adee23 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/averagecolor/AverageColorExtractor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColorExtractor.kt @@ -1,4 +1,4 @@ -package org.vitrivr.engine.module.features.feature.averagecolor +package org.vitrivr.engine.core.features.averagecolor import org.vitrivr.engine.core.features.AbstractExtractor import org.vitrivr.engine.core.features.metadata.source.file.FileSourceMetadataExtractor @@ -39,6 +39,6 @@ class AverageColorExtractor(input: Operator, field: Schema.Field { val content = retrievable.content.filterIsInstance() - return AverageColor().analyse(content).map { it.copy(retrievableId = retrievable.id, field = this@AverageColorExtractor.field) } + return AverageColor.analyse(content).map { it.copy(retrievableId = retrievable.id, field = this@AverageColorExtractor.field) } } } \ No newline at end of file diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/averagecolor/AverageColorRetriever.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColorRetriever.kt similarity index 88% rename from vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/averagecolor/AverageColorRetriever.kt rename to vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColorRetriever.kt index b6d35114..48a788b2 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/averagecolor/AverageColorRetriever.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColorRetriever.kt @@ -1,4 +1,4 @@ -package org.vitrivr.engine.module.features.feature.averagecolor +package org.vitrivr.engine.core.features.averagecolor import io.github.oshai.kotlinlogging.KLogger import io.github.oshai.kotlinlogging.KotlinLogging @@ -20,11 +20,11 @@ import org.vitrivr.engine.core.operators.retrieve.Retriever * @see [AverageColor] * * @author Luca Rossetto - * @version 1.0.0 + * @version 1.1.0 */ class AverageColorRetriever( override val field: Schema.Field, - private val query: ProximityQuery + private val query: ProximityQuery ) : Retriever { private val logger: KLogger = KotlinLogging.logger {} @@ -32,8 +32,8 @@ class AverageColorRetriever( companion object { private const val MAXIMUM_DISTANCE = 3f fun scoringFunction(retrieved: Retrieved): Float { - val distance = retrieved.filteredAttribute()?.distance ?: return 0f - return 1f - (distance / MAXIMUM_DISTANCE) + val distance = retrieved.filteredAttribute()?.distance ?: return 0.0f + return 1.0f - (distance / MAXIMUM_DISTANCE) } } diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/metadata/source/exif/ExifMetadata.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/metadata/source/exif/ExifMetadata.kt index f001c534..f4279824 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/metadata/source/exif/ExifMetadata.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/metadata/source/exif/ExifMetadata.kt @@ -1,16 +1,16 @@ package org.vitrivr.engine.core.features.metadata.source.exif -import io.github.oshai.kotlinlogging.KLogger -import io.github.oshai.kotlinlogging.KotlinLogging import org.vitrivr.engine.core.context.IndexContext import org.vitrivr.engine.core.context.QueryContext import org.vitrivr.engine.core.model.content.element.ContentElement -import org.vitrivr.engine.core.model.descriptor.struct.metadata.source.MapStructDescriptor +import org.vitrivr.engine.core.model.descriptor.Attribute +import org.vitrivr.engine.core.model.descriptor.struct.MapStructDescriptor import org.vitrivr.engine.core.model.metamodel.Analyser import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.query.Query import org.vitrivr.engine.core.model.query.bool.SimpleBooleanQuery import org.vitrivr.engine.core.model.retrievable.Retrievable +import org.vitrivr.engine.core.model.types.Type import org.vitrivr.engine.core.operators.Operator import org.vitrivr.engine.core.operators.ingest.Extractor import org.vitrivr.engine.core.operators.retrieve.Retriever @@ -18,11 +18,17 @@ import java.util.* class ExifMetadata : Analyser, MapStructDescriptor> { - private val logger: KLogger = KotlinLogging.logger {} - override val contentClasses = setOf(ContentElement::class) override val descriptorClass = MapStructDescriptor::class - override fun prototype(field: Schema.Field<*, *>) = MapStructDescriptor.prototype(field.parameters) + override fun prototype(field: Schema.Field<*, *>): MapStructDescriptor { + val parameters = field.parameters.map { (k, v) -> Attribute(k, Type.valueOf(v)) } + return MapStructDescriptor( + UUID.randomUUID(), + UUID.randomUUID(), + parameters, + parameters.associate { it.name to it.type.defaultValue() }, + ) + } override fun newExtractor( name: String, input: Operator, diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/metadata/source/exif/ExifMetadataExtractor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/metadata/source/exif/ExifMetadataExtractor.kt index 07b7f86a..71851c5a 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/metadata/source/exif/ExifMetadataExtractor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/metadata/source/exif/ExifMetadataExtractor.kt @@ -16,7 +16,9 @@ import io.github.oshai.kotlinlogging.KLogger import io.github.oshai.kotlinlogging.KotlinLogging import org.vitrivr.engine.core.features.AbstractExtractor import org.vitrivr.engine.core.model.content.element.ContentElement -import org.vitrivr.engine.core.model.descriptor.struct.metadata.source.MapStructDescriptor +import org.vitrivr.engine.core.model.descriptor.Attribute +import org.vitrivr.engine.core.model.descriptor.AttributeName +import org.vitrivr.engine.core.model.descriptor.struct.MapStructDescriptor import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.retrievable.Retrievable import org.vitrivr.engine.core.model.retrievable.attributes.SourceAttribute @@ -52,36 +54,41 @@ private fun convertDate(date: String): Date? { } -private fun convertType(directory: Directory, tagType: Int, type: Type): Value<*>? = - when (type) { - Type.STRING -> Value.String(directory.getString(tagType)) - Type.BOOLEAN -> Value.Boolean(directory.getBoolean(tagType)) - Type.BYTE -> Value.Byte(directory.getObject(tagType) as Byte) - Type.SHORT -> Value.Short(directory.getObject(tagType) as Short) - Type.INT -> Value.Int(directory.getInt(tagType)) - Type.LONG -> Value.Long(directory.getLong(tagType)) - Type.FLOAT -> Value.Float(directory.getFloat(tagType)) - Type.DOUBLE -> Value.Double(directory.getDouble(tagType)) - Type.DATETIME -> convertDate(directory.getString(tagType))?.let { Value.DateTime(it) } - } +private fun convertType(directory: Directory, tagType: Int, type: Type): Value<*>? = when (type) { + Type.Boolean -> Value.Boolean(directory.getBoolean(tagType)) + Type.Byte -> Value.Byte(directory.getObject(tagType) as Byte) + Type.Datetime -> convertDate(directory.getString(tagType))?.let { Value.DateTime(it) } + Type.Double -> Value.Double(directory.getDouble(tagType)) + Type.Float -> Value.Float(directory.getFloat(tagType)) + Type.Int -> Value.Int(directory.getInt(tagType)) + Type.Long -> Value.Long(directory.getLong(tagType)) + Type.Short -> Value.Short(directory.getObject(tagType) as Short) + Type.String -> Value.String(directory.getString(tagType)) + Type.Text -> Value.String(directory.getString(tagType)) + is Type.BooleanVector -> throw IllegalArgumentException("Unsupported type: $type") + is Type.DoubleVector -> throw IllegalArgumentException("Unsupported type: $type") + is Type.FloatVector -> throw IllegalArgumentException("Unsupported type: $type") + is Type.IntVector -> throw IllegalArgumentException("Unsupported type: $type") + is Type.LongVector -> throw IllegalArgumentException("Unsupported type: $type") +} private fun JsonObject.convertType(type: Type): Value<*>? { val jsonPrimitive = this.jsonPrimitive if (jsonPrimitive.isString) { return when (type) { - Type.STRING -> Value.String(jsonPrimitive.content) - Type.DATETIME -> convertDate(jsonPrimitive.content)?.let { Value.DateTime(it) } + Type.String -> Value.String(jsonPrimitive.content) + Type.Datetime -> convertDate(jsonPrimitive.content)?.let { Value.DateTime(it) } else -> null } } else { return when (type) { - Type.BOOLEAN -> Value.Boolean(jsonPrimitive.boolean) - Type.BYTE -> Value.Byte(jsonPrimitive.int.toByte()) - Type.SHORT -> Value.Short(jsonPrimitive.int.toShort()) - Type.INT -> Value.Int(jsonPrimitive.int) - Type.LONG -> Value.Long(jsonPrimitive.int.toLong()) - Type.FLOAT -> Value.Float(jsonPrimitive.float) - Type.DOUBLE -> Value.Double(jsonPrimitive.double) + Type.Boolean -> Value.Boolean(jsonPrimitive.boolean) + Type.Byte -> Value.Byte(jsonPrimitive.int.toByte()) + Type.Short -> Value.Short(jsonPrimitive.int.toShort()) + Type.Int -> Value.Int(jsonPrimitive.int) + Type.Long -> Value.Long(jsonPrimitive.int.toLong()) + Type.Float -> Value.Float(jsonPrimitive.float) + Type.Double -> Value.Double(jsonPrimitive.double) else -> null } } @@ -98,25 +105,23 @@ class ExifMetadataExtractor( override fun extract(retrievable: Retrievable): List { val metadata = ImageMetadataReader.readMetadata((retrievable.filteredAttribute(SourceAttribute::class.java)?.source as FileSource).path.toFile()) - val columnValues = mutableMapOf>() - - val subfields = this.field!!.parameters - + val columnValues = mutableMapOf>() + val attributes = this.field?.parameters?.map { (k, v) -> k to Attribute(k, Type.valueOf(v)) }?.toMap() ?: emptyMap() for (directory in metadata.directories) { for (tag in directory.tags) { val tagname = tag.tagName.replace(NON_ALPHANUMERIC_REGEX, "") val fullname = "${directory.name.replace(NON_ALPHANUMERIC_REGEX, "")}_$tagname" if (fullname == "ExifSubIFD_UserComment" || fullname == "JpegComment_JPEGComment") { - if (fullname in subfields){ + if (fullname in attributes){ columnValues[fullname] = Value.String(tag.description) } try { val json = Json.parseToJsonElement(tag.description).jsonObject json.forEach { (key, value) -> - subfields[key]?.let { typeString -> - value.jsonObject.convertType(Type.valueOf(typeString))?.let { converted -> + attributes[key]?.let { attribute -> + value.jsonObject.convertType(attribute.type)?.let { converted -> columnValues[key] = converted } } @@ -125,8 +130,8 @@ class ExifMetadataExtractor( logger.warn { "Failed to parse JSON from $fullname: ${tag.description}" } } } else { - subfields[fullname]?.let { typeString -> - convertType(directory, tag.tagType, Type.valueOf(typeString))?.let { converted -> + attributes[fullname]?.let { attribute -> + convertType(directory, tag.tagType, attribute.type)?.let { converted -> columnValues[fullname] = converted } } @@ -136,6 +141,6 @@ class ExifMetadataExtractor( } logger.info { "Extracted fields: ${columnValues.entries.joinToString { (key, value) -> "$key = ${value.value}" }}" } - return listOf(MapStructDescriptor(UUID.randomUUID(), retrievable.id, subfields, columnValues.mapValues { it.value.value }, field = this.field)) + return listOf(MapStructDescriptor(UUID.randomUUID(), retrievable.id, attributes.values.toList(), columnValues.mapValues { it.value }, field = this.field)) } } diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/metadata/source/exif/ExifMetadataRetriever.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/metadata/source/exif/ExifMetadataRetriever.kt index 1ba09ad8..3312c140 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/metadata/source/exif/ExifMetadataRetriever.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/metadata/source/exif/ExifMetadataRetriever.kt @@ -3,7 +3,7 @@ package org.vitrivr.engine.core.features.metadata.source.exif import org.vitrivr.engine.core.context.QueryContext import org.vitrivr.engine.core.features.AbstractRetriever import org.vitrivr.engine.core.model.content.element.ContentElement -import org.vitrivr.engine.core.model.descriptor.struct.metadata.source.MapStructDescriptor +import org.vitrivr.engine.core.model.descriptor.struct.MapStructDescriptor import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.query.Query diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/metadata/source/file/FileSourceMetadataExtractor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/metadata/source/file/FileSourceMetadataExtractor.kt index 3abb8a02..48c46890 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/metadata/source/file/FileSourceMetadataExtractor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/metadata/source/file/FileSourceMetadataExtractor.kt @@ -20,7 +20,7 @@ import kotlin.io.path.absolutePathString * An [Extractor] that extracts [FileSourceMetadataDescriptor]s from [Ingested] objects. * * @author Ralph Gasser - * @version 1.0.0 + * @version 1.1.0 */ class FileSourceMetadataExtractor(input: Operator, field: Schema.Field, FileSourceMetadataDescriptor>?) : AbstractExtractor, FileSourceMetadataDescriptor>(input, field) { /** @@ -47,8 +47,10 @@ class FileSourceMetadataExtractor(input: Operator, field: Schema.Fi FileSourceMetadataDescriptor( id = UUID.randomUUID(), retrievableId = retrievable.id, - path = Value.String(source.path.absolutePathString()), - size = Value.Long(Files.size(source.path)), + mapOf( + "path" to Value.String(source.path.absolutePathString()), + "size" to Value.Long(Files.size(source.path)) + ), this@FileSourceMetadataExtractor.field ) ) diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/metadata/source/video/VideoSourceMetadataExtractor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/metadata/source/video/VideoSourceMetadataExtractor.kt index d8321e1f..8360bc1f 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/metadata/source/video/VideoSourceMetadataExtractor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/metadata/source/video/VideoSourceMetadataExtractor.kt @@ -19,7 +19,7 @@ import java.util.* * An [Extractor] that extracts [VideoSourceMetadataDescriptor]s from [Ingested] objects. * * @author Ralph Gasser - * @version 1.0.0 + * @version 1.1.0 */ class VideoSourceMetadataExtractor(input: Operator, field: Schema.Field, VideoSourceMetadataDescriptor>?) : AbstractExtractor, VideoSourceMetadataDescriptor>(input, field) { /** @@ -46,12 +46,14 @@ class VideoSourceMetadataExtractor(input: Operator, field: Schema.F VideoSourceMetadataDescriptor( id = UUID.randomUUID(), retrievableId = retrievable.id, - width = Value.Int(source.width() ?: 0), - height = Value.Int(source.height() ?: 0), - fps = Value.Double(source.fps() ?: 0.0), - channels = Value.Int(source.channels() ?: 0), - sampleRate = Value.Int(source.sampleRate() ?: 0), - sampleSize = Value.Int(source.sampleSize() ?: 0), + mapOf( + "width" to Value.Int(source.width() ?: 0), + "height" to Value.Int(source.height() ?: 0), + "fps" to Value.Double(source.fps() ?: 0.0), + "channels" to Value.Int(source.channels() ?: 0), + "sampleRate" to Value.Int(source.sampleRate() ?: 0), + "sampleSize" to Value.Int(source.sampleSize() ?: 0) + ), this@VideoSourceMetadataExtractor.field ) ) diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/metadata/temporal/TemporalMetadataExtractor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/metadata/temporal/TemporalMetadataExtractor.kt index 5d9f0b15..f4dee4fe 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/metadata/temporal/TemporalMetadataExtractor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/metadata/temporal/TemporalMetadataExtractor.kt @@ -18,7 +18,7 @@ import java.util.* * An [Extractor] that extracts [TemporalMetadataDescriptor]s from [Ingested] objects. * * @author Ralph Gasser - * @version 1.2.0 + * @version 1.3.0 */ class TemporalMetadataExtractor(input: Operator, field: Schema.Field, TemporalMetadataDescriptor>?) : AbstractExtractor, TemporalMetadataDescriptor>(input, field) { @@ -33,10 +33,10 @@ class TemporalMetadataExtractor(input: Operator, field: Schema.Fiel override fun extract(retrievable: Retrievable): List { if (retrievable.hasAttribute(TimePointAttribute::class.java)) { val timestamp = retrievable.filteredAttribute(TimePointAttribute::class.java)!! - return listOf(TemporalMetadataDescriptor(UUID.randomUUID(), retrievable.id, Value.Long(timestamp.timepointNs), Value.Long(timestamp.timepointNs), this@TemporalMetadataExtractor.field)) + return listOf(TemporalMetadataDescriptor(UUID.randomUUID(), retrievable.id, mapOf("start" to Value.Long(timestamp.timepointNs), "end" to Value.Long(timestamp.timepointNs)), this@TemporalMetadataExtractor.field)) } else if (retrievable.hasAttribute(TimeRangeAttribute::class.java)) { val span = retrievable.filteredAttribute(TimeRangeAttribute::class.java)!! - return listOf(TemporalMetadataDescriptor(UUID.randomUUID(), retrievable.id, Value.Long(span.startNs), Value.Long(span.endNs), this@TemporalMetadataExtractor.field)) + return listOf(TemporalMetadataDescriptor(UUID.randomUUID(), retrievable.id, mapOf("start" to Value.Long(span.startNs), "end" to Value.Long(span.endNs)), this@TemporalMetadataExtractor.field)) } return emptyList() } diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBByteColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBByteColorContainer.kt index 5bdc74f3..8db668c1 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBByteColorContainer.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBByteColorContainer.kt @@ -1,19 +1,45 @@ package org.vitrivr.engine.core.model.color -data class RGBByteColorContainer(val red: UByte, val green: UByte, val blue: UByte) { +/** + * A container for RGB colors. + * + * @version 1.1.0 + * @author Luca Rossetto + * @author Ralph Gasser + */ +@JvmInline +value class RGBByteColorContainer(private val rgb: Int) { + /** + * Constructor for creating a [RGBByteColorContainer] from RGB [UByte] values. + * + * @param red Red component. + * @param green Green component. + * @param blue Blue component. + */ + constructor(red: UByte, green: UByte, blue: UByte) : this(blue.toInt() and 0xFF or ((green.toInt() and 0xFF) shl 8) or ((red.toInt() and 0xFF) shl 16)) - companion object { - fun fromRGB(rgb: Int): RGBByteColorContainer = - RGBByteColorContainer( - (rgb shr 16 and 0xFF).toUByte(), - (rgb shr 8 and 0xFF).toUByte(), - (rgb and 0xFF).toUByte() - ) - } + /** Red component of this [RGBByteColorContainer]. */ + val red: UByte + get() = (this.rgb shr 16 and 0xFF).toUByte() + /** Green component of this [RGBByteColorContainer]. */ + val green: UByte + get() = (this.rgb shr 8 and 0xFF).toUByte() + + /** Blue component of this [RGBByteColorContainer]. */ + val blue: UByte + get() = (this.rgb and 0xFF).toUByte() + + /** + * + */ fun toFloatContainer(): RGBFloatColorContainer = RGBFloatColorContainer(this.red.toFloat() / 255f, this.green.toFloat() / 255f, this.blue.toFloat() / 255f) - fun toRGBInt(): Int = this.blue.toInt() and 0xFF or ((this.green.toInt() and 0xFF) shl 8) or ((this.red.toInt() and 0xFF) shl 16) - + /** + * Converts this [RGBByteColorContainer] to an [Int] representation. + * + * @return [Int] representation of this [RGBByteColorContainer]. + */ + fun toRGBInt(): Int = this.rgb } diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBFloatColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBFloatColorContainer.kt index fe6c35ef..3218bc69 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBFloatColorContainer.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBFloatColorContainer.kt @@ -67,7 +67,7 @@ open class RGBFloatColorContainer(open val red: Float, open val green: Float, op * * @return [List] of [Value.Float]s */ - fun toValueList() = listOf(Value.Float(this.red), Value.Float(this.green), Value.Float(this.blue)) + fun toVector() = Value.FloatVector(floatArrayOf(this.red, this.green, this.blue)) fun distanceTo(other: RGBFloatColorContainer): Float = sqrt( (this.red - other.red) * (this.red - other.red) + (this.green - other.green) * (this.green - other.green) + (this.blue - other.blue) * (this.blue - other.blue) diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/Attribute.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/Attribute.kt new file mode 100644 index 00000000..df1a6930 --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/Attribute.kt @@ -0,0 +1,15 @@ +package org.vitrivr.engine.core.model.descriptor + +import org.vitrivr.engine.core.model.types.Type + +/** The name of an attribute. */ +typealias AttributeName = String + +/** + * A [Attribute] describes a field in a [Descriptor]. This is required for a [Descriptor]'s ability to + * describe its own schema. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +data class Attribute(val name: AttributeName, val type: Type, val nullable: Boolean = false) \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/Descriptor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/Descriptor.kt index 73a953cb..bfbe2939 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/Descriptor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/Descriptor.kt @@ -4,6 +4,7 @@ import org.vitrivr.engine.core.model.Persistable import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.retrievable.Retrievable import org.vitrivr.engine.core.model.retrievable.RetrievableId +import org.vitrivr.engine.core.model.types.Value import java.util.* /** A typealias to identify the [UUID] identifying a [Descriptor]. */ @@ -31,16 +32,16 @@ interface Descriptor : Persistable { get() = this.field != null /** - * Returns a [FieldSchema] [List] for this [Descriptor]. + * Returns a [Attribute] [List] for this [Descriptor]. * - * @return [List] of [FieldSchema] + * @return [List] of [Attribute] */ - fun schema(): List + fun layout(): List /** * Returns the fields and its values of this [Descriptor] as a [Map]. * * @return A [Map] of this [Descriptor]'s fields (without the IDs). */ - fun values(): List> + fun values(): Map?> } \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/FieldSchema.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/FieldSchema.kt deleted file mode 100644 index baa6529a..00000000 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/FieldSchema.kt +++ /dev/null @@ -1,12 +0,0 @@ -package org.vitrivr.engine.core.model.descriptor - -import org.vitrivr.engine.core.model.types.Type - -/** - * A [FieldSchema] describes a field in a [Descriptor]. This is required for a [Descriptor]'s ability to - * describe its own schema. - * - * @author Ralph Gasser - * @version 1.0.0 - */ -data class FieldSchema(val name: String, val type: Type, val dimensions: IntArray = intArrayOf(), val nullable: Boolean = false) \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/BooleanDescriptor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/BooleanDescriptor.kt index e1334a2a..356b582a 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/BooleanDescriptor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/BooleanDescriptor.kt @@ -1,7 +1,8 @@ package org.vitrivr.engine.core.model.descriptor.scalar +import org.vitrivr.engine.core.model.descriptor.Attribute import org.vitrivr.engine.core.model.descriptor.DescriptorId -import org.vitrivr.engine.core.model.descriptor.FieldSchema +import org.vitrivr.engine.core.model.descriptor.scalar.ScalarDescriptor.Companion.VALUE_ATTRIBUTE_NAME import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.retrievable.RetrievableId import org.vitrivr.engine.core.model.types.Type @@ -21,13 +22,13 @@ data class BooleanDescriptor( override val field: Schema.Field<*, BooleanDescriptor>? = null ) : ScalarDescriptor { companion object { - private val SCHEMA = listOf(FieldSchema("value", Type.BOOLEAN)) + private val SCHEMA = listOf(Attribute(VALUE_ATTRIBUTE_NAME, Type.Boolean)) } /** - * Returns the [FieldSchema] [List ]of this [BooleanDescriptor]. + * Returns the [Attribute] [List ]of this [BooleanDescriptor]. * - * @return [List] of [FieldSchema] + * @return [List] of [Attribute] */ - override fun schema(): List = SCHEMA + override fun layout(): List = SCHEMA } \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/DoubleDescriptor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/DoubleDescriptor.kt index 8cb8e234..828408f4 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/DoubleDescriptor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/DoubleDescriptor.kt @@ -1,7 +1,8 @@ package org.vitrivr.engine.core.model.descriptor.scalar +import org.vitrivr.engine.core.model.descriptor.Attribute import org.vitrivr.engine.core.model.descriptor.DescriptorId -import org.vitrivr.engine.core.model.descriptor.FieldSchema +import org.vitrivr.engine.core.model.descriptor.scalar.ScalarDescriptor.Companion.VALUE_ATTRIBUTE_NAME import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.retrievable.RetrievableId import org.vitrivr.engine.core.model.types.Type @@ -21,13 +22,13 @@ data class DoubleDescriptor( override val field: Schema.Field<*, DoubleDescriptor>? = null ) : ScalarDescriptor { companion object { - private val SCHEMA = listOf(FieldSchema("value", Type.DOUBLE)) + private val SCHEMA = listOf(Attribute(VALUE_ATTRIBUTE_NAME, Type.Double)) } /** - * Returns the [FieldSchema] [List] of this [DoubleDescriptor]. + * Returns the [Attribute] [List] of this [DoubleDescriptor]. * - * @return [List] of [FieldSchema] + * @return [List] of [Attribute] */ - override fun schema(): List = SCHEMA + override fun layout(): List = SCHEMA } \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/FloatDescriptor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/FloatDescriptor.kt index f0f0bf7f..b87fa80c 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/FloatDescriptor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/FloatDescriptor.kt @@ -1,7 +1,8 @@ package org.vitrivr.engine.core.model.descriptor.scalar +import org.vitrivr.engine.core.model.descriptor.Attribute import org.vitrivr.engine.core.model.descriptor.DescriptorId -import org.vitrivr.engine.core.model.descriptor.FieldSchema +import org.vitrivr.engine.core.model.descriptor.scalar.ScalarDescriptor.Companion.VALUE_ATTRIBUTE_NAME import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.retrievable.RetrievableId import org.vitrivr.engine.core.model.types.Type @@ -21,13 +22,13 @@ data class FloatDescriptor( override val field: Schema.Field<*, FloatDescriptor>? = null ) : ScalarDescriptor { companion object { - private val SCHEMA = listOf(FieldSchema("value", Type.FLOAT)) + private val SCHEMA = listOf(Attribute(VALUE_ATTRIBUTE_NAME, Type.Float)) } /** - * Returns the [FieldSchema] [List] of this [FloatDescriptor]. + * Returns the [Attribute] [List] of this [FloatDescriptor]. * - * @return [List] of [FieldSchema] + * @return [List] of [Attribute] */ - override fun schema(): List = SCHEMA + override fun layout(): List = SCHEMA } \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/IntDescriptor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/IntDescriptor.kt index cbfb3ba8..b56daff3 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/IntDescriptor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/IntDescriptor.kt @@ -1,7 +1,8 @@ package org.vitrivr.engine.core.model.descriptor.scalar +import org.vitrivr.engine.core.model.descriptor.Attribute import org.vitrivr.engine.core.model.descriptor.DescriptorId -import org.vitrivr.engine.core.model.descriptor.FieldSchema +import org.vitrivr.engine.core.model.descriptor.scalar.ScalarDescriptor.Companion.VALUE_ATTRIBUTE_NAME import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.retrievable.RetrievableId import org.vitrivr.engine.core.model.types.Type @@ -21,13 +22,13 @@ data class IntDescriptor( override val field: Schema.Field<*, IntDescriptor>? = null ) : ScalarDescriptor { companion object { - private val SCHEMA = listOf(FieldSchema("value", Type.INT)) + private val SCHEMA = listOf(Attribute(VALUE_ATTRIBUTE_NAME, Type.Int)) } /** - * Returns the [FieldSchema] [List] of this [IntDescriptor]. + * Returns the [Attribute] [List] of this [IntDescriptor]. * - * @return [List] of [FieldSchema] + * @return [List] of [Attribute] */ - override fun schema(): List = SCHEMA + override fun layout(): List = SCHEMA } \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/LongDescriptor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/LongDescriptor.kt index a9a56df2..a472150d 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/LongDescriptor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/LongDescriptor.kt @@ -1,7 +1,8 @@ package org.vitrivr.engine.core.model.descriptor.scalar +import org.vitrivr.engine.core.model.descriptor.Attribute import org.vitrivr.engine.core.model.descriptor.DescriptorId -import org.vitrivr.engine.core.model.descriptor.FieldSchema +import org.vitrivr.engine.core.model.descriptor.scalar.ScalarDescriptor.Companion.VALUE_ATTRIBUTE_NAME import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.retrievable.RetrievableId import org.vitrivr.engine.core.model.types.Type @@ -21,13 +22,13 @@ data class LongDescriptor( override val field: Schema.Field<*, LongDescriptor>? = null ) : ScalarDescriptor { companion object { - private val SCHEMA = listOf(FieldSchema("value", Type.LONG)) + private val SCHEMA = listOf(Attribute(VALUE_ATTRIBUTE_NAME, Type.Long)) } /** - * Returns the [FieldSchema] [List] of this [LongDescriptor]. + * Returns the [Attribute] [List] of this [LongDescriptor]. * - * @return [List] of [FieldSchema] + * @return [List] of [Attribute] */ - override fun schema(): List = SCHEMA + override fun layout(): List = SCHEMA } \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/ScalarDescriptor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/ScalarDescriptor.kt index 6be50862..452a369f 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/ScalarDescriptor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/ScalarDescriptor.kt @@ -1,5 +1,6 @@ package org.vitrivr.engine.core.model.descriptor.scalar +import org.vitrivr.engine.core.model.descriptor.AttributeName import org.vitrivr.engine.core.model.descriptor.Descriptor import org.vitrivr.engine.core.model.types.Value @@ -7,9 +8,13 @@ import org.vitrivr.engine.core.model.types.Value * A [Descriptor] with a scalar value [T]. * * @author Ralph Gasser - * @version 1.0.0 + * @version 1.1.0 */ -sealed interface ScalarDescriptor> : Descriptor { +sealed interface ScalarDescriptor> : Descriptor { + + companion object { + const val VALUE_ATTRIBUTE_NAME = "value" + } /** The [Value] held by this [ScalarDescriptor]. */ val value: T @@ -19,5 +24,5 @@ sealed interface ScalarDescriptor> : Descriptor { * * @return A [Map] of this [ScalarDescriptor]'s fields (without the IDs). */ - override fun values(): List> = listOf("value" to this.value) + override fun values(): Map = mapOf("value" to this.value) } diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/StringDescriptor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/StringDescriptor.kt index e6615886..3a1bfa07 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/StringDescriptor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/scalar/StringDescriptor.kt @@ -1,7 +1,8 @@ package org.vitrivr.engine.core.model.descriptor.scalar +import org.vitrivr.engine.core.model.descriptor.Attribute import org.vitrivr.engine.core.model.descriptor.DescriptorId -import org.vitrivr.engine.core.model.descriptor.FieldSchema +import org.vitrivr.engine.core.model.descriptor.scalar.ScalarDescriptor.Companion.VALUE_ATTRIBUTE_NAME import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.retrievable.RetrievableId import org.vitrivr.engine.core.model.types.Type @@ -21,13 +22,13 @@ data class StringDescriptor( override val field: Schema.Field<*, StringDescriptor>? = null ) : ScalarDescriptor { companion object { - private val SCHEMA = listOf(FieldSchema("value", Type.STRING)) + private val SCHEMA = listOf(Attribute(VALUE_ATTRIBUTE_NAME, Type.String)) } /** - * Returns the [FieldSchema] [List ]of this [StringDescriptor]. + * Returns the [Attribute] [List ]of this [StringDescriptor]. * - * @return [List] of [FieldSchema] + * @return [List] of [Attribute] */ - override fun schema(): List = SCHEMA + override fun layout(): List = SCHEMA } diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/LabelDescriptor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/LabelDescriptor.kt index 125b5c6b..33e7896e 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/LabelDescriptor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/LabelDescriptor.kt @@ -1,7 +1,8 @@ package org.vitrivr.engine.core.model.descriptor.struct import org.vitrivr.engine.core.model.descriptor.DescriptorId -import org.vitrivr.engine.core.model.descriptor.FieldSchema +import org.vitrivr.engine.core.model.descriptor.Attribute +import org.vitrivr.engine.core.model.descriptor.AttributeName import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.retrievable.RetrievableId import org.vitrivr.engine.core.model.types.Type @@ -11,31 +12,24 @@ import org.vitrivr.engine.core.model.types.Value * A [StructDescriptor] used to store a string label and a confidence for that label. * * @author Ralph Gasser - * @version 1.0.0 + * @version 2.0.0 */ -data class LabelDescriptor( +class LabelDescriptor( override var id: DescriptorId, override var retrievableId: RetrievableId?, - val label: Value.String, - val confidence: Value.Float, + values: Map?>, override val field: Schema.Field<*, LabelDescriptor>? = null -) : StructDescriptor { +) : MapStructDescriptor(id, retrievableId, SCHEMA, values, field) { companion object { private val SCHEMA = listOf( - FieldSchema("label", Type.STRING), - FieldSchema("confidence", Type.FLOAT), + Attribute("label", Type.String), + Attribute("confidence", Type.Float), ) } - /** - * Returns the [FieldSchema] [List ]of this [LabelDescriptor]. - * - * @return [List] of [FieldSchema] - */ - override fun schema(): List = SCHEMA + /** The stored label. */ + val label: Value.String by this.values - override fun values(): List> = listOf( - "label" to this.label, - "confidence" to this.confidence - ) + /** The associated confidence. */ + val confidence: Value.Float by this.values } diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/MapStructDescriptor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/MapStructDescriptor.kt new file mode 100644 index 00000000..bf12ba56 --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/MapStructDescriptor.kt @@ -0,0 +1,37 @@ +package org.vitrivr.engine.core.model.descriptor.struct + +import org.vitrivr.engine.core.model.descriptor.Attribute +import org.vitrivr.engine.core.model.descriptor.AttributeName +import org.vitrivr.engine.core.model.descriptor.Descriptor +import org.vitrivr.engine.core.model.descriptor.DescriptorId +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.retrievable.RetrievableId +import org.vitrivr.engine.core.model.types.Value + +/** + * An abstract [StructDescriptor] implementation that is backed by a [Map] of [Value]s. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +open class MapStructDescriptor( + override var id: DescriptorId, + override var retrievableId: RetrievableId?, + protected val layout: List, + protected val values: Map?>, + override val field: Schema.Field<*, out StructDescriptor>? = null +) : StructDescriptor { + /** + * Returns a [Attribute] [List] for this [Descriptor]. + * + * @return [List] of [Attribute] + */ + final override fun layout(): List = this.layout + + /** + * Returns the fields and its values of this [Descriptor] as a [Map]. + * + * @return A [Map] of this [Descriptor]'s fields (without the IDs). + */ + final override fun values(): Map?> = this.values +} \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/RasterDescriptor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/RasterDescriptor.kt deleted file mode 100644 index 72b89c5a..00000000 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/RasterDescriptor.kt +++ /dev/null @@ -1,33 +0,0 @@ -package org.vitrivr.engine.core.model.descriptor.struct - -import org.vitrivr.engine.core.model.descriptor.DescriptorId -import org.vitrivr.engine.core.model.descriptor.FieldSchema -import org.vitrivr.engine.core.model.metamodel.Schema -import org.vitrivr.engine.core.model.retrievable.RetrievableId -import org.vitrivr.engine.core.model.types.Type -import org.vitrivr.engine.core.model.types.Value - -data class RasterDescriptor( - override var id: DescriptorId, - override var retrievableId: RetrievableId?, - val hist: List, - val raster: List, - override val field: Schema.Field<*, RasterDescriptor>? = null -) : StructDescriptor { - - - /** - * Returns the [FieldSchema] [List ]of this [RasterDescriptor]. - * - * @return [List] of [FieldSchema] - */ - override fun schema(): List = listOf( - FieldSchema("hist", Type.FLOAT, intArrayOf(this.hist.size)), - FieldSchema("raster", Type.FLOAT, intArrayOf(this.raster.size)) - ) - - override fun values(): List> = listOf( - "hist" to this.hist, - "raster" to this.raster - ) -} \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/SkeletonDescriptor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/SkeletonDescriptor.kt deleted file mode 100644 index 8d3ec7d9..00000000 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/SkeletonDescriptor.kt +++ /dev/null @@ -1,35 +0,0 @@ -package org.vitrivr.engine.core.model.descriptor.struct - -import org.vitrivr.engine.core.model.descriptor.DescriptorId -import org.vitrivr.engine.core.model.descriptor.FieldSchema -import org.vitrivr.engine.core.model.metamodel.Schema -import org.vitrivr.engine.core.model.retrievable.RetrievableId -import org.vitrivr.engine.core.model.types.Type -import org.vitrivr.engine.core.model.types.Value - -data class SkeletonDescriptor( - override var id: DescriptorId, - override var retrievableId: RetrievableId?, - val person: Value.Int, - val skeleton: List, - val weights: List, - override val field: Schema.Field<*, SkeletonDescriptor>? = null -) : StructDescriptor { - - /** - * Returns the [FieldSchema] [List ]of this [SkeletonDescriptor]. - * - * @return [List] of [FieldSchema] - */ - override fun schema(): List = listOf( - FieldSchema("person", Type.INT), - FieldSchema("skeleton", Type.FLOAT, intArrayOf(this.skeleton.size)), - FieldSchema("weights", Type.FLOAT, intArrayOf(this.weights.size)) - ) - - override fun values(): List> = listOf( - "person" to this.person, - "skeleton" to this.skeleton, - "weights" to this.weights - ) -} \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/metadata/MediaDimensionsDescriptor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/metadata/MediaDimensionsDescriptor.kt index de28fa22..f38028ba 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/metadata/MediaDimensionsDescriptor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/metadata/MediaDimensionsDescriptor.kt @@ -1,7 +1,9 @@ package org.vitrivr.engine.core.model.descriptor.struct.metadata import org.vitrivr.engine.core.model.descriptor.DescriptorId -import org.vitrivr.engine.core.model.descriptor.FieldSchema +import org.vitrivr.engine.core.model.descriptor.Attribute +import org.vitrivr.engine.core.model.descriptor.AttributeName +import org.vitrivr.engine.core.model.descriptor.struct.MapStructDescriptor import org.vitrivr.engine.core.model.descriptor.struct.StructDescriptor import org.vitrivr.engine.core.model.descriptor.struct.metadata.source.VideoSourceMetadataDescriptor import org.vitrivr.engine.core.model.metamodel.Schema @@ -14,41 +16,35 @@ import java.util.* * A [StructDescriptor] used to store metadata about a 2D raster graphic. * * @author Ralph Gasser - * @version 1.0.0 + * @version 2.0.0 */ -data class MediaDimensionsDescriptor( +class MediaDimensionsDescriptor( override var id: DescriptorId, override var retrievableId: RetrievableId?, - val width: Value.Int, - val height: Value.Int, + values: Map?>, override val field: Schema.Field<*, MediaDimensionsDescriptor>? = null -) : StructDescriptor { +) : MapStructDescriptor(id, retrievableId, SCHEMA, values, field) { + companion object { + /** The field schema associated with a [VideoSourceMetadataDescriptor]. */ + private val SCHEMA = listOf( + Attribute("width", Type.Int), + Attribute("height", Type.Int), + ) - companion object { - /** The field schema associated with a [VideoSourceMetadataDescriptor]. */ - private val SCHEMA = listOf( - FieldSchema("width", Type.INT), - FieldSchema("height", Type.INT), + /** The prototype [MediaDimensionsDescriptor]. */ + val PROTOTYPE = MediaDimensionsDescriptor( + UUID.randomUUID(), + UUID.randomUUID(), + mapOf( + "width" to Value.Int(0), + "height" to Value.Int(0) ) + ) + } - /** The prototype [MediaDimensionsDescriptor]. */ - val PROTOTYPE = MediaDimensionsDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.Int(0), Value.Int(0)) - } - - /** - * Returns the [FieldSchema] [List ]of this [StructDescriptor]. - * - * @return [List] of [FieldSchema] - */ - override fun schema(): List = SCHEMA + /** The width of this image. */ + val width: Value.Int by this.values - /** - * Returns the fields and its values of this [MediaDimensionsDescriptor] as a [Map]. - * - * @return A [Map] of this [MediaDimensionsDescriptor]'s fields (without the IDs). - */ - override fun values(): List> = listOf( - "width" to this.width, - "height" to this.height - ) + /** The height of this image. */ + val height: Value.Int by this.values } \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/metadata/Rectangle2DMetadataDescriptor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/metadata/Rectangle2DMetadataDescriptor.kt index 70c4fa32..acba69ce 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/metadata/Rectangle2DMetadataDescriptor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/metadata/Rectangle2DMetadataDescriptor.kt @@ -1,7 +1,9 @@ package org.vitrivr.engine.core.model.descriptor.struct.metadata import org.vitrivr.engine.core.model.descriptor.DescriptorId -import org.vitrivr.engine.core.model.descriptor.FieldSchema +import org.vitrivr.engine.core.model.descriptor.Attribute +import org.vitrivr.engine.core.model.descriptor.AttributeName +import org.vitrivr.engine.core.model.descriptor.struct.MapStructDescriptor import org.vitrivr.engine.core.model.descriptor.struct.StructDescriptor import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.retrievable.RetrievableId @@ -13,47 +15,46 @@ import java.util.* * A [StructDescriptor] used to store spatial metadata in a 2D raster graphic. * * @author Ralph Gasser - * @version 1.0.0 + * @version 2.0.0 */ -data class Rectangle2DMetadataDescriptor( +class Rectangle2DMetadataDescriptor( override var id: DescriptorId, override var retrievableId: RetrievableId?, - val leftX: Value.Int, - val leftY: Value.Int, - val width: Value.Int, - val height: Value.Int, + values: Map?>, override val field: Schema.Field<*, Rectangle2DMetadataDescriptor>? = null -) : StructDescriptor { +) : MapStructDescriptor(id, retrievableId, SCHEMA, values, field) { companion object { /** The field schema associated with a [Rectangle2DMetadataDescriptor]. */ private val SCHEMA = listOf( - FieldSchema("leftX", Type.INT), - FieldSchema("leftY", Type.INT), - FieldSchema("width", Type.INT), - FieldSchema("height", Type.INT), + Attribute("leftX", Type.Int), + Attribute("leftY", Type.Int), + Attribute("width", Type.Int), + Attribute("height", Type.Int), ) /** The prototype [Rectangle2DMetadataDescriptor]. */ - val PROTOTYPE = Rectangle2DMetadataDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.Int(0), Value.Int(0), Value.Int(0), Value.Int(0)) + val PROTOTYPE = Rectangle2DMetadataDescriptor( + UUID.randomUUID(), + UUID.randomUUID(), + mapOf( + "leftX" to Value.Int(0), + "leftY" to Value.Int(0), + "width" to Value.Int(0), + "height" to Value.Int(0) + ) + ) } - /** - * Returns the [FieldSchema] [List ]of this [StructDescriptor]. - * - * @return [List] of [FieldSchema] - */ - override fun schema(): List = SCHEMA - - /** - * Returns the fields and its values of this [Rectangle2DMetadataDescriptor] as a [Map]. - * - * @return A [Map] of this [Rectangle2DMetadataDescriptor]'s fields (without the IDs). - */ - override fun values(): List> = listOf( - "leftX" to this.leftX, - "leftY" to this.leftY, - "width" to this.width, - "height" to this.height - ) + /** The top left pixel of the rectangle (X-component). */ + val leftX: Value.Int by this.values + + /** The top left pixel of the rectangle (Y-component). */ + val leftY: Value.Int by this.values + + /** The width of the rectangle. */ + val width: Value.Int by this.values + + /** The height of the rectangle. */ + val height: Value.Int by this.values } \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/metadata/TemporalMetadataDescriptor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/metadata/TemporalMetadataDescriptor.kt index fd3b36f0..8f027226 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/metadata/TemporalMetadataDescriptor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/metadata/TemporalMetadataDescriptor.kt @@ -1,7 +1,9 @@ package org.vitrivr.engine.core.model.descriptor.struct.metadata import org.vitrivr.engine.core.model.descriptor.DescriptorId -import org.vitrivr.engine.core.model.descriptor.FieldSchema +import org.vitrivr.engine.core.model.descriptor.Attribute +import org.vitrivr.engine.core.model.descriptor.AttributeName +import org.vitrivr.engine.core.model.descriptor.struct.MapStructDescriptor import org.vitrivr.engine.core.model.descriptor.struct.StructDescriptor import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.retrievable.RetrievableId @@ -13,41 +15,29 @@ import java.util.* * A [StructDescriptor] used to store temporal metadata. * * @author Ralph Gasser - * @version 1.0.0 + * @version 2.0.0 */ -data class TemporalMetadataDescriptor( +class TemporalMetadataDescriptor( override var id: DescriptorId, override var retrievableId: RetrievableId?, //retrievable Id must come first, due to reflection - val startNs: Value.Long, - val endNs: Value.Long, + values: Map?>, override val field: Schema.Field<*, TemporalMetadataDescriptor>? = null -) : StructDescriptor { +) : MapStructDescriptor(id, retrievableId, SCHEMA, values, field) { companion object { /** The field schema associated with a [TemporalMetadataDescriptor]. */ private val SCHEMA = listOf( - FieldSchema("start", Type.LONG), - FieldSchema("end", Type.LONG), + Attribute("start", Type.Long), + Attribute("end", Type.Long), ) /** The prototype [TemporalMetadataDescriptor]. */ - val PROTOTYPE = TemporalMetadataDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.Long(0L), Value.Long(0L)) + val PROTOTYPE = TemporalMetadataDescriptor(UUID.randomUUID(), UUID.randomUUID(), mapOf("start" to Value.Long(0L), "end" to Value.Long(0L))) } - /** - * Returns the [FieldSchema] [List ]of this [TemporalMetadataDescriptor]. - * - * @return [List] of [FieldSchema] - */ - override fun schema(): List = SCHEMA + /** The start timestamp in nanoseconds. */ + val start: Value.Long by this.values - /** - * Returns the fields and its values of this [TemporalMetadataDescriptor] as a [Map]. - * - * @return A [Map] of this [TemporalMetadataDescriptor]'s fields (without the IDs). - */ - override fun values(): List> = listOf( - "start" to this.startNs, - "end" to this.endNs - ) + /** The end timestamp in nanoseconds. */ + val end: Value.Long by this.values } \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/metadata/source/FileSourceMetadataDescriptor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/metadata/source/FileSourceMetadataDescriptor.kt index 1de8e8c1..273f5159 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/metadata/source/FileSourceMetadataDescriptor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/metadata/source/FileSourceMetadataDescriptor.kt @@ -1,7 +1,9 @@ package org.vitrivr.engine.core.model.descriptor.struct.metadata.source +import org.vitrivr.engine.core.model.descriptor.Attribute +import org.vitrivr.engine.core.model.descriptor.AttributeName import org.vitrivr.engine.core.model.descriptor.DescriptorId -import org.vitrivr.engine.core.model.descriptor.FieldSchema +import org.vitrivr.engine.core.model.descriptor.struct.MapStructDescriptor import org.vitrivr.engine.core.model.descriptor.struct.StructDescriptor import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.retrievable.RetrievableId @@ -13,41 +15,32 @@ import java.util.* * A [StructDescriptor] used to store metadata about a file. * * @author Ralph Gasser - * @version 1.0.0 + * @version 2.0.0 */ -data class FileSourceMetadataDescriptor( +class FileSourceMetadataDescriptor( override var id: DescriptorId, override var retrievableId: RetrievableId?, - val path: Value.String, - val size: Value.Long, + values: Map?>, override val field: Schema.Field<*, FileSourceMetadataDescriptor>? = null -) : StructDescriptor { +) : MapStructDescriptor(id, retrievableId, SCHEMA, values, field) { + + constructor(id: DescriptorId, retrievableId: RetrievableId?, path: Value.String, size: Value.Long, field: Schema.Field<*, FileSourceMetadataDescriptor>) : + this(id, retrievableId, mapOf("path" to path, "size" to size), field) + + /** The path to the file. */ + val path: Value.String by this.values + + /** The size of the file in bytes. */ + val size: Value.Long by this.values companion object { /** The field schema associated with a [FileSourceMetadataDescriptor]. */ private val SCHEMA = listOf( - FieldSchema("path", Type.STRING), - FieldSchema("size", Type.LONG), + Attribute("path", Type.String), + Attribute("size", Type.Long), ) /** The prototype [FileSourceMetadataDescriptor]. */ - val PROTOTYPE = FileSourceMetadataDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.String(""), Value.Long(0L)) + val PROTOTYPE = FileSourceMetadataDescriptor(UUID.randomUUID(), UUID.randomUUID(), mapOf("path" to Value.String(""), "size" to Value.Long(0L))) } - - /** - * Returns the [FieldSchema] [List ]of this [StructDescriptor]. - * - * @return [List] of [FieldSchema] - */ - override fun schema(): List = SCHEMA - - /** - * Returns the fields and its values of this [FileSourceMetadataDescriptor] as a [Map]. - * - * @return A [Map] of this [FileSourceMetadataDescriptor]'s fields (without the IDs). - */ - override fun values(): List> = listOf( - "path" to this.path, - "size" to this.size - ) } \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/metadata/source/MapStructDescriptor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/metadata/source/MapStructDescriptor.kt deleted file mode 100644 index 53e94935..00000000 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/metadata/source/MapStructDescriptor.kt +++ /dev/null @@ -1,65 +0,0 @@ -package org.vitrivr.engine.core.model.descriptor.struct.metadata.source - -import org.vitrivr.engine.core.model.descriptor.DescriptorId -import org.vitrivr.engine.core.model.descriptor.FieldSchema -import org.vitrivr.engine.core.model.descriptor.struct.StructDescriptor -import org.vitrivr.engine.core.model.metamodel.Schema -import org.vitrivr.engine.core.model.retrievable.RetrievableId -import org.vitrivr.engine.core.model.types.Type -import org.vitrivr.engine.core.model.types.Value -import java.util.* - -data class MapStructDescriptor( - override var id: DescriptorId, - override var retrievableId: RetrievableId?, - val columnTypes: Map, - val columnValues: Map, - override val transient: Boolean = false, - override val field: Schema.Field<*, MapStructDescriptor>? = null -) : StructDescriptor { - - companion object{ - fun prototype(columnTypes: Map): MapStructDescriptor { - val columnValues = columnTypes.mapValues { (_, type) -> - when (Type.valueOf(type)) { - Type.STRING -> "" - Type.BOOLEAN -> false - Type.BYTE -> 0.toByte() - Type.SHORT -> 0.toShort() - Type.INT -> 0 - Type.LONG -> 0L - Type.FLOAT -> 0.0f - Type.DOUBLE -> 0.0 - Type.DATETIME -> Date() - } - } - return MapStructDescriptor(UUID.randomUUID(), UUID.randomUUID(), columnTypes, columnValues) - - } - } - - override fun schema(): List { - return this.columnTypes.map { (key, type) -> - FieldSchema(key, Type.valueOf(type), nullable=true) - } - } - - override fun values(): List> { - return this.columnTypes.map { (key, type) -> - val value = this.columnValues[key] // This will be null if key is not present in columnValues - val pairedValue = when (Type.valueOf(type)) { - Type.STRING -> (value as? Value.String) - Type.BOOLEAN -> (value as? Value.Boolean) - Type.BYTE -> (value as? Value.Byte) - Type.SHORT -> (value as? Value.Short) - Type.INT -> (value as? Value.Int) - Type.LONG -> (value as? Value.Long) - Type.FLOAT -> (value as? Value.Float) - Type.DOUBLE -> (value as? Value.Double) - Type.DATETIME -> (value as? Value.DateTime) - } - Pair(key, pairedValue) - } - - } -} diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/metadata/source/VideoSourceMetadataDescriptor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/metadata/source/VideoSourceMetadataDescriptor.kt index 9b226f10..248741d7 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/metadata/source/VideoSourceMetadataDescriptor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/struct/metadata/source/VideoSourceMetadataDescriptor.kt @@ -1,7 +1,9 @@ package org.vitrivr.engine.core.model.descriptor.struct.metadata.source import org.vitrivr.engine.core.model.descriptor.DescriptorId -import org.vitrivr.engine.core.model.descriptor.FieldSchema +import org.vitrivr.engine.core.model.descriptor.Attribute +import org.vitrivr.engine.core.model.descriptor.AttributeName +import org.vitrivr.engine.core.model.descriptor.struct.MapStructDescriptor import org.vitrivr.engine.core.model.descriptor.struct.StructDescriptor import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.retrievable.RetrievableId @@ -13,53 +15,56 @@ import java.util.* * A [StructDescriptor] used to store metadata about a video source (e.g., a file). * * @author Ralph Gasser - * @version 1.0.0 + * @version 2.0.0 */ class VideoSourceMetadataDescriptor( override var id: DescriptorId, override var retrievableId: RetrievableId?, - val width: Value.Int, - val height: Value.Int, - val fps: Value.Double, - val channels: Value.Int, - val sampleRate: Value.Int, - val sampleSize: Value.Int, + values: Map?>, override val field: Schema.Field<*, VideoSourceMetadataDescriptor>? = null -) : StructDescriptor { +) : MapStructDescriptor(id, retrievableId, SCHEMA, values, field) { + + /** The width of the video source in pixels. */ + val width: Value.Int by this.values + + /** The width of the video source in pixels. */ + val height: Value.Int by this.values + + /** The number of visual frames per seconds. */ + val fps: Value.Double by this.values + + /** The number of audio channels. */ + val channels: Value.Int by this.values + + /** The sample rate in kHz. */ + val sampleRate: Value.Int by this.values + + /** The sample size in bytes. */ + val sampleSize: Value.Int by this.values companion object { /** The field schema associated with a [VideoSourceMetadataDescriptor]. */ private val SCHEMA = listOf( - FieldSchema("width", Type.INT), - FieldSchema("height", Type.INT), - FieldSchema("fps", Type.DOUBLE), - FieldSchema("channels", Type.INT), - FieldSchema("sampleRate", Type.INT), - FieldSchema("sampleSize", Type.INT), + Attribute("width", Type.Int), + Attribute("height", Type.Int), + Attribute("fps", Type.Double), + Attribute("channels", Type.Int), + Attribute("sampleRate", Type.Int), + Attribute("sampleSize", Type.Int), ) /** The prototype [VideoSourceMetadataDescriptor]. */ - val PROTOTYPE = VideoSourceMetadataDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.Int(0), Value.Int(0), Value.Double(0.0), Value.Int(0), Value.Int(0), Value.Int(0)) + val PROTOTYPE = VideoSourceMetadataDescriptor( + UUID.randomUUID(), + UUID.randomUUID(), + mapOf( + "width" to Value.Int(0), + "height" to Value.Int(0), + "fps" to Value.Double(0.0), + "channels" to Value.Int(0), + "sampleRate" to Value.Int(0), + "sampleSize" to Value.Int(0) + ) + ) } - - /** - * Returns the [FieldSchema] [List ]of this [StructDescriptor]. - * - * @return [List] of [FieldSchema] - */ - override fun schema(): List = SCHEMA - - /** - * Returns the fields and its values of this [FileSourceMetadataDescriptor] as a [Map]. - * - * @return A [Map] of this [FileSourceMetadataDescriptor]'s fields (without the IDs). - */ - override fun values(): List> = listOf( - "width" to this.width, - "height" to this.height, - "fps" to this.fps, - "channels" to this.channels, - "sampleRate" to this.sampleRate, - "sampleSize" to this.sampleSize - ) } \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/vector/BooleanVectorDescriptor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/vector/BooleanVectorDescriptor.kt index d5dcc468..52b29910 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/vector/BooleanVectorDescriptor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/vector/BooleanVectorDescriptor.kt @@ -1,6 +1,7 @@ package org.vitrivr.engine.core.model.descriptor.vector -import org.vitrivr.engine.core.model.descriptor.FieldSchema +import org.vitrivr.engine.core.model.descriptor.Attribute +import org.vitrivr.engine.core.model.descriptor.vector.VectorDescriptor.Companion.VECTOR_ATTRIBUTE_NAME import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.retrievable.RetrievableId import org.vitrivr.engine.core.model.types.Type @@ -12,19 +13,19 @@ import java.util.* * * @author Luca Rossetto * @author Ralph Gasser - * @version 1.0.0 + * @version 1.1.0 */ data class BooleanVectorDescriptor( override var id: UUID = UUID.randomUUID(), override var retrievableId: RetrievableId? = null, - override val vector: List, + override val vector: Value.BooleanVector, override val field: Schema.Field<*, BooleanVectorDescriptor>? = null -) : VectorDescriptor { +) : VectorDescriptor { /** - * Returns the [FieldSchema] [List ]of this [BooleanVectorDescriptor]. + * Returns the [Attribute] [List ]of this [BooleanVectorDescriptor]. * - * @return [List] of [FieldSchema] + * @return [List] of [Attribute] */ - override fun schema(): List = listOf(FieldSchema("vector", Type.BOOLEAN, intArrayOf(this.dimensionality))) + override fun layout(): List = listOf(Attribute(VECTOR_ATTRIBUTE_NAME, Type.BooleanVector(this.dimensionality))) } diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/vector/DoubleVectorDescriptor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/vector/DoubleVectorDescriptor.kt index 46ec0081..a5c1f028 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/vector/DoubleVectorDescriptor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/vector/DoubleVectorDescriptor.kt @@ -1,6 +1,7 @@ package org.vitrivr.engine.core.model.descriptor.vector -import org.vitrivr.engine.core.model.descriptor.FieldSchema +import org.vitrivr.engine.core.model.descriptor.Attribute +import org.vitrivr.engine.core.model.descriptor.vector.VectorDescriptor.Companion.VECTOR_ATTRIBUTE_NAME import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.retrievable.RetrievableId import org.vitrivr.engine.core.model.types.Type @@ -12,19 +13,19 @@ import java.util.* * * @author Luca Rossetto * @author Ralph Gasser - * @version 1.0.0 + * @version 1.1.0 */ data class DoubleVectorDescriptor( override var id: UUID = UUID.randomUUID(), override var retrievableId: RetrievableId? = null, - override val vector: List, + override val vector: Value.DoubleVector, override val field: Schema.Field<*, DoubleVectorDescriptor>? = null -) : VectorDescriptor { +) : VectorDescriptor { /** - * Returns the [FieldSchema] [List ]of this [DoubleVectorDescriptor]. + * Returns the [Attribute] [List ]of this [DoubleVectorDescriptor]. * - * @return [List] of [FieldSchema] + * @return [List] of [Attribute] */ - override fun schema(): List = listOf(FieldSchema("vector", Type.DOUBLE, intArrayOf(this.dimensionality))) + override fun layout(): List = listOf(Attribute(VECTOR_ATTRIBUTE_NAME, Type.DoubleVector(this.dimensionality))) } diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/vector/FloatVectorDescriptor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/vector/FloatVectorDescriptor.kt index 6e9f4ace..d1205f52 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/vector/FloatVectorDescriptor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/vector/FloatVectorDescriptor.kt @@ -1,6 +1,7 @@ package org.vitrivr.engine.core.model.descriptor.vector -import org.vitrivr.engine.core.model.descriptor.FieldSchema +import org.vitrivr.engine.core.model.descriptor.Attribute +import org.vitrivr.engine.core.model.descriptor.vector.VectorDescriptor.Companion.VECTOR_ATTRIBUTE_NAME import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.retrievable.RetrievableId import org.vitrivr.engine.core.model.types.Type @@ -13,19 +14,19 @@ import java.util.* * * @author Luca Rossetto * @author Ralph Gasser - * @version 1.0.0 + * @version 1.1.0 */ data class FloatVectorDescriptor( override var id: UUID = UUID.randomUUID(), override var retrievableId: RetrievableId? = null, - override val vector: List, + override val vector: Value.FloatVector, override val field: Schema.Field<*, FloatVectorDescriptor>? = null -) : VectorDescriptor { +) : VectorDescriptor { /** - * Returns the [FieldSchema] [List ]of this [FloatVectorDescriptor]. + * Returns the [Attribute] [List ]of this [FloatVectorDescriptor]. * - * @return [List] of [FieldSchema] + * @return [List] of [Attribute] */ - override fun schema(): List = listOf(FieldSchema("vector", Type.FLOAT, intArrayOf(this.dimensionality))) + override fun layout(): List = listOf(Attribute(VECTOR_ATTRIBUTE_NAME, Type.FloatVector(this.dimensionality))) } diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/vector/IntVectorDescriptor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/vector/IntVectorDescriptor.kt index 4b3b42ee..dbea9d17 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/vector/IntVectorDescriptor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/vector/IntVectorDescriptor.kt @@ -1,6 +1,7 @@ package org.vitrivr.engine.core.model.descriptor.vector -import org.vitrivr.engine.core.model.descriptor.FieldSchema +import org.vitrivr.engine.core.model.descriptor.Attribute +import org.vitrivr.engine.core.model.descriptor.vector.VectorDescriptor.Companion.VECTOR_ATTRIBUTE_NAME import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.retrievable.RetrievableId import org.vitrivr.engine.core.model.types.Type @@ -12,19 +13,19 @@ import java.util.* * * @author Luca Rossetto * @author Ralph Gasser - * @version 1.0.0 + * @version 1.1.0 */ data class IntVectorDescriptor( override var id: UUID = UUID.randomUUID(), override var retrievableId: RetrievableId? = null, - override val vector: List, + override val vector: Value.IntVector, override val field: Schema.Field<*, IntVectorDescriptor>? = null -) : VectorDescriptor { +) : VectorDescriptor { /** - * Returns the [FieldSchema] [List ]of this [IntVectorDescriptor]. + * Returns the [Attribute] [List ]of this [IntVectorDescriptor]. * - * @return [List] of [FieldSchema] + * @return [List] of [Attribute] */ - override fun schema(): List = listOf(FieldSchema("vector", Type.INT, intArrayOf(this.dimensionality))) + override fun layout(): List = listOf(Attribute(VECTOR_ATTRIBUTE_NAME, Type.Int)) } diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/vector/LongVectorDescriptor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/vector/LongVectorDescriptor.kt index 74d41884..825f0226 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/vector/LongVectorDescriptor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/vector/LongVectorDescriptor.kt @@ -1,6 +1,7 @@ package org.vitrivr.engine.core.model.descriptor.vector -import org.vitrivr.engine.core.model.descriptor.FieldSchema +import org.vitrivr.engine.core.model.descriptor.Attribute +import org.vitrivr.engine.core.model.descriptor.vector.VectorDescriptor.Companion.VECTOR_ATTRIBUTE_NAME import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.retrievable.RetrievableId import org.vitrivr.engine.core.model.types.Type @@ -12,19 +13,19 @@ import java.util.* * * @author Luca Rossetto * @author Ralph Gasser - * @version 1.0.0 + * @version 1.1.0 */ data class LongVectorDescriptor( override var id: UUID = UUID.randomUUID(), override var retrievableId: RetrievableId? = null, - override val vector: List, + override val vector: Value.LongVector, override val field: Schema.Field<*, LongVectorDescriptor>? = null -) : VectorDescriptor { +) : VectorDescriptor { /** - * Returns the [FieldSchema] [List ]of this [LongVectorDescriptor]. + * Returns the [Attribute] [List ]of this [LongVectorDescriptor]. * - * @return [List] of [FieldSchema] + * @return [List] of [Attribute] */ - override fun schema(): List = listOf(FieldSchema("vector", Type.LONG, intArrayOf(this.dimensionality))) + override fun layout(): List = listOf(Attribute(VECTOR_ATTRIBUTE_NAME, Type.Long)) } diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/vector/VectorDescriptor.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/vector/VectorDescriptor.kt index ffbcc1be..bb02b8a4 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/vector/VectorDescriptor.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/descriptor/vector/VectorDescriptor.kt @@ -1,5 +1,6 @@ package org.vitrivr.engine.core.model.descriptor.vector +import org.vitrivr.engine.core.model.descriptor.AttributeName import org.vitrivr.engine.core.model.descriptor.Descriptor import org.vitrivr.engine.core.model.descriptor.scalar.ScalarDescriptor import org.vitrivr.engine.core.model.types.Value @@ -9,20 +10,24 @@ import org.vitrivr.engine.core.model.types.Value * * @author Luca Rossetto * @author Ralph Gasser - * @version 1.0.0 + * @version 1.1.0 */ -sealed interface VectorDescriptor> : Descriptor { +sealed interface VectorDescriptor> : Descriptor { + companion object { + const val VECTOR_ATTRIBUTE_NAME = "vector" + } + /** The size of this [VectorDescriptor]. */ val dimensionality: Int get() = this.vector.size /** The [List] of values [T]. */ - val vector: List + val vector: T /** * Returns the fields and its values of this [ScalarDescriptor] as a [Map]. * * @return A [Map] of this [ScalarDescriptor]'s fields (without the IDs). */ - override fun values(): List?>> = listOf("vector" to this.vector) + override fun values(): Map = mapOf(VECTOR_ATTRIBUTE_NAME to this.vector) } \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/metamodel/Schema.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/metamodel/Schema.kt index 68add8d6..a0dacb11 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/metamodel/Schema.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/metamodel/Schema.kt @@ -2,6 +2,7 @@ package org.vitrivr.engine.core.model.metamodel import org.vitrivr.engine.core.config.ingest.IngestionConfig import org.vitrivr.engine.core.config.ingest.IngestionPipelineBuilder +import org.vitrivr.engine.core.config.schema.IndexConfig import org.vitrivr.engine.core.config.schema.SchemaConfig import org.vitrivr.engine.core.context.IndexContext import org.vitrivr.engine.core.context.QueryContext @@ -29,7 +30,7 @@ typealias FieldName = String * A [Schema] that defines a particular vitrivr instance's meta data model. * * @author Ralph Gasser - * @version 1.0.0 + * @version 1.1.0 */ open class Schema(val name: String = "vitrivr", val connection: Connection) : Closeable { @@ -49,13 +50,13 @@ open class Schema(val name: String = "vitrivr", val connection: Connection) : Cl * Adds a new [Field] to this [Schema]. * * @param name The name of the new [Field]. Must be unique. + * @param analyser The [Analyser] to use with the new [Field]. + * @param parameters The (optional) parameters used to configure the [Field]. + * @param indexes List of [IndexConfig]s that can be used to configure indexes on the [Field]. + * @return [Field] instance. */ - fun addField( - name: String, - analyser: Analyser, Descriptor>, - parameters: Map = emptyMap() - ) { - this.fields.add(Field(name, analyser, parameters)) + fun addField(name: String, analyser: Analyser, Descriptor>, parameters: Map = emptyMap(), indexes: List) { + this.fields.add(Field(name, analyser, parameters, indexes)) } /** @@ -157,8 +158,14 @@ open class Schema(val name: String = "vitrivr", val connection: Connection) : Cl inner class Field, D : Descriptor>( val fieldName: FieldName, val analyser: Analyser, - val parameters: Map = emptyMap() + val parameters: Map = emptyMap(), + val indexes: List = emptyList() ) { + + init { + require(this.fieldName.matches(Regex("^[a-zA-Z0-9]+$"))) { "Field name can only contain alpha-numeric characters and numbers." } + } + /** Pointer to the [Schema] this [Field] belongs to.*/ val schema: Schema get() = this@Schema diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/metamodel/SchemaManager.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/metamodel/SchemaManager.kt index 7c994c0e..cc221d86 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/metamodel/SchemaManager.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/metamodel/SchemaManager.kt @@ -21,7 +21,7 @@ import kotlin.concurrent.write * The [SchemaManager] maps [Schema] definitions to database [Connection] objects. * * @author Ralph Gasser - * @version 1.0.0 + * @version 1.0.1 */ class SchemaManager { /** An internal [HashMap] of all open [Connection]. */ @@ -55,7 +55,7 @@ class SchemaManager { throw IllegalArgumentException("Field names must not have a dot (.) in their name.") } @Suppress("UNCHECKED_CAST") - schema.addField(it.name, analyser as Analyser, Descriptor>, it.parameters) + schema.addField(it.name, analyser as Analyser, Descriptor>, it.parameters, it.indexes) } config.resolvers.map { schema.addResolver(it.key, (loadServiceForName(it.value.factory) ?: throw IllegalArgumentException("Failed to find resolver factory implementation for '${it.value.factory}'.")).newResolver(schema, it.value.parameters)) diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/query/basics/Distance.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/query/basics/Distance.kt index 5c90b1bb..4e6c5156 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/query/basics/Distance.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/query/basics/Distance.kt @@ -1,6 +1,10 @@ package org.vitrivr.engine.core.model.query.basics import org.vitrivr.engine.core.model.query.proximity.ProximityQuery +import org.vitrivr.engine.core.model.types.Value +import kotlin.math.abs +import kotlin.math.pow +import kotlin.math.sqrt /** * Enumeration of [Distance] functions supported by [ProximityQuery]. @@ -9,7 +13,106 @@ import org.vitrivr.engine.core.model.query.proximity.ProximityQuery * @version 1.0.0 */ enum class Distance { - MANHATTAN, - EUCLIDEAN, - COSINE; + MANHATTAN { + override fun invoke(v1: Value.FloatVector, v2: Value.FloatVector): Float { + var sum = 0.0f + for (i in v1.value.indices) { + sum += abs(v1.value[i] - v2.value[i]) + } + return sum + } + + override fun invoke(v1: Value.DoubleVector, v2: Value.DoubleVector): Double { + var sum = 0.0 + for (i in v1.value.indices) { + sum += abs(v1.value[i] - v2.value[i]) + } + return sum + } + }, + EUCLIDEAN { + override fun invoke(v1: Value.FloatVector, v2: Value.FloatVector): Float { + var sum = 0.0f + for (i in v1.value.indices) { + sum += (v1.value[i] - v2.value[i]).pow(2) + } + return sqrt(sum) + } + + override fun invoke(v1: Value.DoubleVector, v2: Value.DoubleVector): Double { + var sum = 0.0 + for (i in v1.value.indices) { + sum += (v1.value[i] - v2.value[i]).pow(2) + } + return sqrt(sum) + } + }, + COSINE { + override fun invoke(v1: Value.FloatVector, v2: Value.FloatVector): Float { + var dotProduct = 0.0f + var normV1 = 0.0f + var normV2 = 0.0f + for (i in v1.value.indices) { + dotProduct += v1.value[i] * v2.value[i] + normV1 += v1.value[i].pow(2) + normV2 += v2.value[i].pow(2) + } + normV1 = sqrt(normV1) + normV2 = sqrt(normV2) + return -(dotProduct / (normV1 * normV2)) + } + + override fun invoke(v1: Value.DoubleVector, v2: Value.DoubleVector): Double { + var dotProduct = 0.0 + var normV1 = 0.0 + var normV2 = 0.0 + for (i in v1.value.indices) { + dotProduct += v1.value[i] * v2.value[i] + normV1 += v1.value[i].pow(2) + normV2 += v2.value[i].pow(2) + } + normV1 = sqrt(normV1) + normV2 = sqrt(normV2) + return -(dotProduct / (normV1 * normV2)) + } + }, + HAMMING { + override fun invoke(v1: Value.FloatVector, v2: Value.FloatVector): Float { + var sum = 0.0f + for (i in v1.value.indices) { + sum += if (v1.value[i] != v2.value[i]) 1.0f else 0.0f + } + return sum + } + + override fun invoke(v1: Value.DoubleVector, v2: Value.DoubleVector): Double { + var sum = 0.0 + for (i in v1.value.indices) { + sum += if (v1.value[i] != v2.value[i]) 1.0 else 0.0 + } + return sum + } + }, + JACCARD { + override fun invoke(v1: Value.FloatVector, v2: Value.FloatVector): Float = throw UnsupportedOperationException("Jaccard distance is not supported for float vectors.") + override fun invoke(v1: Value.DoubleVector, v2: Value.DoubleVector): Double = throw UnsupportedOperationException("Jaccard distance is not supported for float vectors.") + }; + + /** + * Calculates this [Distance] between two [Value.FloatVector]. + * + * @param v1 [Value.FloatVector] First vector for distance calculation. + * @param v2 [Value.FloatVector] Second vector for distance calculation. + * @return [Float] + */ + abstract operator fun invoke(v1: Value.FloatVector, v2: Value.FloatVector): Float + + /** + * Calculates this [Distance] between two [Value.DoubleVector]. + * + * @param v1 [Value.DoubleVector] First vector for distance calculation. + * @param v2 [Value.DoubleVector] Second vector for distance calculation. + * @return [Double] + */ + abstract operator fun invoke(v1: Value.DoubleVector, v2: Value.DoubleVector): Double } \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/query/proximity/ProximityQuery.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/query/proximity/ProximityQuery.kt index 8088a933..8456ad87 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/query/proximity/ProximityQuery.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/query/proximity/ProximityQuery.kt @@ -17,9 +17,9 @@ import org.vitrivr.engine.core.model.types.Value * @version 1.0.0 */ -data class ProximityQuery>( +data class ProximityQuery>( /** The [VectorDescriptor] being used; specifies both the query field and the comparison value. */ - val value: List, + val value: T, /** The [Distance] used for the comparison. */ val distance: Distance = Distance.EUCLIDEAN, diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/retrievable/attributes/DistanceAttribute.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/retrievable/attributes/DistanceAttribute.kt index 9298841a..4531ea05 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/retrievable/attributes/DistanceAttribute.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/retrievable/attributes/DistanceAttribute.kt @@ -2,6 +2,12 @@ package org.vitrivr.engine.core.model.retrievable.attributes import kotlin.math.min +/** + * A [MergingRetrievableAttribute] that contains a distance value. + * + * @author Luca Rossetto + * @version 1.1.0 + */ data class DistanceAttribute(val distance: Float) : MergingRetrievableAttribute { override fun merge(other: MergingRetrievableAttribute): DistanceAttribute = DistanceAttribute( min(this.distance, (other as? DistanceAttribute)?.distance ?: Float.POSITIVE_INFINITY) diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/types/Type.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/types/Type.kt index 131f7ce8..8f3928aa 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/types/Type.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/types/Type.kt @@ -1,20 +1,171 @@ package org.vitrivr.engine.core.model.types +import java.util.* + /** - * An enumeration of supported [Type]s + * The [Type]s supported by vitrivr-engine. * * @author Ralph Gasser * @author Loris Sauter - * @version 1.1.0 + * @version 2.0.0 */ -enum class Type { - STRING, - BOOLEAN, - BYTE, - SHORT, - INT, - LONG, - FLOAT, - DOUBLE, - DATETIME +sealed interface Type { + + companion object { + /** + * Returns the [Type] for the given [String] representation. + * + * @param type [String] representation of the [Type] + * @return [Type] + */ + fun valueOf(type: kotlin.String, dimensions: kotlin.Int = 0): Type { + return when (type.uppercase()) { + "TEXT" -> Text + "STRING" -> String + "BOOLEAN" -> Boolean + "BYTE" -> Byte + "SHORT" -> Short + "INT" -> Int + "LONG" -> Long + "FLOAT" -> Float + "DOUBLE" -> Double + "DATETIME" -> Datetime + "BOOLEANVECTOR" -> BooleanVector(dimensions) + "INTVECTOR" -> IntVector(dimensions) + "LONGVECTOR" -> LongVector(dimensions) + "FLOATVECTOR" -> FloatVector(dimensions) + "DOUBLEVECTOR" -> DoubleVector(dimensions) + else -> throw IllegalArgumentException("Type $type is not supported!") + } + } + } + + + /** The number of dimensions for this [Type]. */ + val dimensions: kotlin.Int + + + /** + * + */ + fun defaultValue(): Value<*> + + /** + * A [Type] that represents a (longer) text. + * + * Primarily needed, since simple strings and longer texts are treated differently by certain databases. + */ + data object Text : Type { + override val dimensions: kotlin.Int = 1 + override fun defaultValue(): Value<*> = Value.String("") + } + + /** + * A [Type] that represents a simple String. + * + * Primarily needed, since simple strings and longer texts are treated differently by certain databases. + */ + data object String : Type { + override val dimensions: kotlin.Int = 1 + override fun defaultValue(): Value<*> = Value.String("") + } + + /** + * A [Type] that represents a [Boolean] value. + */ + data object Boolean : Type { + override val dimensions: kotlin.Int = 1 + override fun defaultValue(): Value<*> = Value.Boolean(false) + } + + /** + * A [Type] that represents a [Byte] value. + */ + data object Byte : Type { + override val dimensions: kotlin.Int = 1 + override fun defaultValue(): Value<*> = Value.Byte(0) + } + + /** + * A [Type] that represents a [Short] value. + */ + data object Short : Type { + override val dimensions: kotlin.Int = 1 + override fun defaultValue(): Value<*> = Value.Short(0) + } + + /** + * A [Type] that represents a [Int] value. + */ + data object Int : Type { + override val dimensions: kotlin.Int = 1 + override fun defaultValue(): Value<*> = Value.Int(0) + } + + /** + * A [Type] that represents a [Long] value. + */ + data object Long : Type { + override val dimensions: kotlin.Int = 1 + override fun defaultValue(): Value<*> = Value.Long(0L) + } + + /** + * A [Type] that represents a [Float] value. + */ + data object Float : Type { + override val dimensions: kotlin.Int = 1 + override fun defaultValue(): Value<*> = Value.Float(0.0f) + } + + /** + * A [Type] that represents a [Double] value. + */ + data object Double : Type { + override val dimensions: kotlin.Int = 1 + override fun defaultValue(): Value<*> = Value.Double(0.0) + } + + /** + * A [Type] that represents a [Datetime] value. + */ + data object Datetime : Type { + override val dimensions: kotlin.Int = 1 + override fun defaultValue(): Value<*> = Value.DateTime(Date()) + } + + /** + * A [Type] that represents a [BooleanVector] value. + */ + data class BooleanVector(override val dimensions: kotlin.Int) : Type { + override fun defaultValue(): Value<*> = Value.BooleanVector(this.dimensions) + } + + /** + * A [Type] that represents a [IntVector] value. + */ + data class IntVector(override val dimensions: kotlin.Int) : Type { + override fun defaultValue(): Value<*> = Value.IntVector(this.dimensions) + } + + /** + * A [Type] that represents a [LongVector] value. + */ + data class LongVector(override val dimensions: kotlin.Int) : Type { + override fun defaultValue(): Value<*> = Value.LongVector(this.dimensions) + } + + /** + * A [Type] that represents a [FloatVector] value. + */ + data class FloatVector(override val dimensions: kotlin.Int) : Type { + override fun defaultValue(): Value<*> = Value.FloatVector(this.dimensions) + } + + /** + * A [Type] that represents a [DoubleVector] value. + */ + data class DoubleVector(override val dimensions: kotlin.Int) : Type { + override fun defaultValue(): Value<*> = Value.DoubleVector(this.dimensions) + } } diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/types/Value.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/types/Value.kt index a0e3f37e..5bd1dbbb 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/types/Value.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/types/Value.kt @@ -1,6 +1,6 @@ package org.vitrivr.engine.core.model.types -import java.util.Date +import java.util.* /** * A [Value] in vitrivr-engine maps primitive data types. @@ -13,6 +13,12 @@ import java.util.Date sealed interface Value { companion object { + /** + * Converts [Any] value to a [Value]. + * + * @param value The value to convert. + * @return [Value] + */ fun of(value: Any): Value<*> = when (value) { is kotlin.String -> String(value) is kotlin.Boolean -> Boolean(value) @@ -22,37 +28,106 @@ sealed interface Value { is kotlin.Long -> Long(value) is kotlin.Float -> Float(value) is kotlin.Double -> Double(value) + is BooleanArray -> BooleanVector(value) + is DoubleArray -> DoubleVector(value) + is FloatArray -> FloatVector(value) + is LongArray -> LongVector(value) + is IntArray -> IntVector(value) is Date -> DateTime(value) else -> throw IllegalArgumentException("Unsupported data type.") } } + /** The actual, primitive value. */ val value: T + + sealed interface ScalarValue: Value + + @JvmInline + value class String(override val value: kotlin.String) : ScalarValue + @JvmInline - value class String(override val value: kotlin.String) : Value + value class Text(override val value: kotlin.String) : ScalarValue @JvmInline - value class Boolean(override val value: kotlin.Boolean) : Value + value class Boolean(override val value: kotlin.Boolean) : ScalarValue @JvmInline - value class Byte(override val value: kotlin.Byte) : Value + value class Byte(override val value: kotlin.Byte) : ScalarValue @JvmInline - value class Short(override val value: kotlin.Short) : Value + value class Short(override val value: kotlin.Short) : ScalarValue @JvmInline - value class Int(override val value: kotlin.Int) : Value + value class Int(override val value: kotlin.Int) : ScalarValue @JvmInline - value class Long(override val value: kotlin.Long) : Value + value class Long(override val value: kotlin.Long) : ScalarValue @JvmInline - value class Float(override val value: kotlin.Float) : Value + value class Float(override val value: kotlin.Float) : ScalarValue @JvmInline - value class Double(override val value: kotlin.Double) : Value + value class Double(override val value: kotlin.Double) : ScalarValue @JvmInline - value class DateTime(override val value: Date) : Value + value class DateTime(override val value: Date) : ScalarValue + + /** + * A [Vector] in vitrivr-engine maps primitive data types. + * + * Part of the vitrivr-engine's internal type system. + * + * @param T The type of the elements in the [Vector]. + * @property size The size of the [Vector]. + * @property value The actual, primitive value. + * @constructor Creates a new [Vector] with the given [value]. + */ + sealed interface Vector : Value { + val size: kotlin.Int + } + + + @JvmInline + value class BooleanVector(override val value: BooleanArray) : Vector { + constructor(size: kotlin.Int, init: (kotlin.Int) -> kotlin.Boolean = { false }) : this(BooleanArray(size, init)) + + override val size: kotlin.Int + get() = this.value.size + } + + @JvmInline + value class IntVector(override val value: IntArray) : Vector { + constructor(size: kotlin.Int, init: (kotlin.Int) -> kotlin.Int = { 0 }) : this(IntArray(size, init)) + + override val size: kotlin.Int + get() = this.value.size + } + + + @JvmInline + value class LongVector(override val value: LongArray) : Vector { + constructor(size: kotlin.Int, init: (kotlin.Int) -> kotlin.Long = { 0L }) : this(LongArray(size, init)) + + override val size: kotlin.Int + get() = this.value.size + } + + + @JvmInline + value class FloatVector(override val value: FloatArray) : Vector { + constructor(size: kotlin.Int, init: (kotlin.Int) -> kotlin.Float = { 0.0f }) : this(FloatArray(size, init)) + + override val size: kotlin.Int + get() = this.value.size + } + + @JvmInline + value class DoubleVector(override val value: DoubleArray) : Vector { + constructor(size: kotlin.Int, init: (kotlin.Int) -> kotlin.Double = { 0.0 }) : this(DoubleArray(size, init)) + + override val size: kotlin.Int + get() = this.value.size + } } diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/operators/persistence/PersistingSink.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/operators/persistence/PersistingSink.kt index e84a0466..9fb7c824 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/operators/persistence/PersistingSink.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/operators/persistence/PersistingSink.kt @@ -61,11 +61,13 @@ class PersistingSink(override val input: Operator, val context: Ind collect(retrievable, Triple(retrievables, relationships, descriptors)) /* Write entities to database. */ - this.writer.addAll(retrievables) - this.writer.connectAll(relationships) - for ((f, d) in descriptors) { - val writer = f.let { field -> this.descriptorWriters.computeIfAbsent(field) { it.getWriter() } } as? DescriptorWriter - writer?.addAll(d) + this.writer.connection.withTransaction { + this.writer.addAll(retrievables) + this.writer.connectAll(relationships) + for ((f, d) in descriptors) { + val writer = f.let { field -> this.descriptorWriters.computeIfAbsent(field) { it.getWriter() } } as? DescriptorWriter + writer?.addAll(d) + } } } diff --git a/vitrivr-engine-core/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser b/vitrivr-engine-core/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser index a8bdfd01..736805a4 100644 --- a/vitrivr-engine-core/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser +++ b/vitrivr-engine-core/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser @@ -1,3 +1,4 @@ +org.vitrivr.engine.core.features.averagecolor.AverageColor org.vitrivr.engine.core.features.metadata.source.file.FileSourceMetadata org.vitrivr.engine.core.features.metadata.source.exif.ExifMetadata org.vitrivr.engine.core.features.metadata.source.video.VideoSourceMetadata diff --git a/vitrivr-engine-core/src/test/kotlin/org/vitrivr/engine/core/database/MockConnection.kt b/vitrivr-engine-core/src/test/kotlin/org/vitrivr/engine/core/database/MockConnection.kt index b3b37405..6d682b49 100755 --- a/vitrivr-engine-core/src/test/kotlin/org/vitrivr/engine/core/database/MockConnection.kt +++ b/vitrivr-engine-core/src/test/kotlin/org/vitrivr/engine/core/database/MockConnection.kt @@ -9,12 +9,14 @@ import org.vitrivr.engine.core.database.retrievable.RetrievableWriter import org.vitrivr.engine.core.model.descriptor.Descriptor import org.vitrivr.engine.core.model.metamodel.Schema -class MockConnection( - schemaName: String, - provider: ConnectionProvider = MockConnectionProvider(schemaName) -) : AbstractConnection(schemaName, provider) { +class MockConnection(schemaName: String, provider: ConnectionProvider = MockConnectionProvider(schemaName)) : AbstractConnection(schemaName, provider) { override val provider: ConnectionProvider get() = TODO("Not yet implemented") + + override fun withTransaction(action: (Unit) -> T): T { + TODO("Not yet implemented") + } + override val schemaName: String get() = TODO("Not yet implemented") diff --git a/vitrivr-engine-core/src/testFixtures/kotlin/org/vitrivr/engine/core/database/AbstractDatabaseTest.kt b/vitrivr-engine-core/src/testFixtures/kotlin/org/vitrivr/engine/core/database/AbstractDatabaseTest.kt new file mode 100644 index 00000000..2952a11a --- /dev/null +++ b/vitrivr-engine-core/src/testFixtures/kotlin/org/vitrivr/engine/core/database/AbstractDatabaseTest.kt @@ -0,0 +1,32 @@ +package org.vitrivr.engine.core.database + +import org.vitrivr.engine.core.config.schema.SchemaConfig +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.metamodel.SchemaManager + +/** + * Abstract base class for database tests. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +abstract class AbstractDatabaseTest(schemaPath: String) { + companion object { + val SCHEMA_NAME = "vitrivr-test" + } + + /** [SchemaManager] for this [AbstractDatabaseTest]. */ + protected val manager = SchemaManager() + + init { + /* Loads schema. */ + val schema = SchemaConfig.loadFromResource(schemaPath) + this.manager.load(schema) + } + + /** The test [Schema]. */ + protected val testSchema: Schema = this.manager.getSchema("vitrivr-test") ?: throw IllegalArgumentException("Schema 'vitrivr-test' not found!") + + /** The [Connection] object backing the [Schema]. */ + protected val testConnection = this.testSchema.connection +} \ No newline at end of file diff --git a/vitrivr-engine-core/src/testFixtures/kotlin/org/vitrivr/engine/core/database/descriptor/AbstractDescriptorInitializerTest.kt b/vitrivr-engine-core/src/testFixtures/kotlin/org/vitrivr/engine/core/database/descriptor/AbstractDescriptorInitializerTest.kt new file mode 100644 index 00000000..cf1570e1 --- /dev/null +++ b/vitrivr-engine-core/src/testFixtures/kotlin/org/vitrivr/engine/core/database/descriptor/AbstractDescriptorInitializerTest.kt @@ -0,0 +1,93 @@ +package org.vitrivr.engine.core.database.descriptor + +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.MethodSource +import org.vitrivr.engine.core.database.AbstractDatabaseTest +import org.vitrivr.engine.core.database.retrievable.RetrievableInitializer +import org.vitrivr.engine.core.model.metamodel.Schema +import java.util.stream.Stream + +/** + * An abstract test to test the functionality of the [DescriptorInitializer]. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +abstract class AbstractDescriptorInitializerTest(schemaPath: String) : AbstractDatabaseTest(schemaPath) { + /** + * Tests if the [DescriptorInitializer.isInitialized] works as expected., + */ + @ParameterizedTest + @MethodSource("getFields") + fun testIsInitializedWithoutInitialization(field: Schema.Field<*, *>) { + Assertions.assertFalse(this.testConnection.getDescriptorInitializer(field).isInitialized()) + } + + /** + * Tests if the [RetrievableInitializer] can be initialized without throwing an exception. Furthermore, + * the test should check if the necessary tables have been created. + */ + @ParameterizedTest + @MethodSource("getFields") + fun testInitializeEntities(field: Schema.Field<*, *>) { + /* Check initialization status (should be false). */ + Assertions.assertFalse(this.testConnection.getDescriptorInitializer(field).isInitialized()) + + /* Initialize basic tables. */ + this.testConnection.getDescriptorInitializer(field).initialize() + + /* Check initialization status (should be true). */ + Assertions.assertTrue(this.testConnection.getDescriptorInitializer(field).isInitialized()) + } + + /** + * Tests if the [RetrievableInitializer] can be initialized without throwing an exception. Furthermore, + * the test should check if the necessary tables have been created. + */ + @ParameterizedTest + @MethodSource("getFields") + fun testDeInitializeEntities(field: Schema.Field<*, *>) { + /* Check initialization status (should be false). */ + Assertions.assertFalse(this.testConnection.getDescriptorInitializer(field).isInitialized()) + + /* Initialize basic tables. */ + this.testConnection.getDescriptorInitializer(field).initialize() + + /* Check initialization status (should be true). */ + Assertions.assertTrue(this.testConnection.getDescriptorInitializer(field).isInitialized()) + + this.testConnection.getDescriptorInitializer(field).deinitialize() + Assertions.assertFalse(this.testConnection.getDescriptorInitializer(field).isInitialized()) + } + + /** + * Prepares the database before each test. + */ + @BeforeEach + open fun prepare() { + this.testSchema.connection.getRetrievableInitializer().initialize() + } + + /** + * Cleans up the database after each test. + */ + @AfterEach + open fun cleanup() { + this.testSchema.connection.getRetrievableInitializer().deinitialize() + for (field in this.testSchema.fields()) { + this.testSchema.connection.getDescriptorInitializer(field).deinitialize() + } + } + + /** + * Returns a [Stream] of [Schema.Field]s that are part of the test schema. + * + * @return [Stream] of [Schema.Field]s + */ + private fun getFields(): Stream> = this.testSchema.fields().map { it as Schema.Field<*, *> }.stream() +} \ No newline at end of file diff --git a/vitrivr-engine-core/src/testFixtures/kotlin/org/vitrivr/engine/core/database/descriptor/struct/AbstractFileMetadataDescriptorReaderTest.kt b/vitrivr-engine-core/src/testFixtures/kotlin/org/vitrivr/engine/core/database/descriptor/struct/AbstractFileMetadataDescriptorReaderTest.kt new file mode 100644 index 00000000..b4504161 --- /dev/null +++ b/vitrivr-engine-core/src/testFixtures/kotlin/org/vitrivr/engine/core/database/descriptor/struct/AbstractFileMetadataDescriptorReaderTest.kt @@ -0,0 +1,254 @@ +package org.vitrivr.engine.core.database.descriptor.struct + +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.vitrivr.engine.core.database.AbstractDatabaseTest +import org.vitrivr.engine.core.database.descriptor.DescriptorWriter +import org.vitrivr.engine.core.model.descriptor.struct.metadata.source.FileSourceMetadataDescriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.query.basics.ComparisonOperator +import org.vitrivr.engine.core.model.query.bool.SimpleBooleanQuery +import org.vitrivr.engine.core.model.query.fulltext.SimpleFulltextQuery +import org.vitrivr.engine.core.model.retrievable.Ingested +import org.vitrivr.engine.core.model.types.Value +import java.nio.file.Paths +import java.util.* + +/** + * An [AbstractDatabaseTest] that tests for basic boolean queries on [FileSourceMetadataDescriptor]. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +abstract class AbstractFileMetadataDescriptorReaderTest(schemaPath: String) : AbstractDatabaseTest(schemaPath) { + + companion object { + /* Define a list of possible root directories. */ + val ROOTS = arrayOf( + "/home", + "/home/test", + "/home/user/", + "/home/admin/", + "/tmp/", + "/var/log/", + "/var/", + "/usr/local/bin/", + "/var/opt", + "/etc", + "/tank", + "/mnt/tank", + "/usr", + "/usr/lib" + ) + + /* Define a list of possible root directories. */ + val FILENAMES = listOf( + "file1.txt", + "test.txt", + "document.pdf", + "image.png", + "image.jpg", + "image.jpeg", + "notes.docx", + "video.mp4", + "audio.mp3", + "test.jpg", + "test.png" + ) + } + + + /** The [Schema.Field] used for this [AbstractFileMetadataDescriptorReaderTest]. */ + private val field: Schema.Field<*, FileSourceMetadataDescriptor> = this.testSchema["file"]!! as Schema.Field<*, FileSourceMetadataDescriptor> + + /** + * Tests for equals comparison. + */ + @Test + fun testBooleanQueryEquals() { + val writer = this.testConnection.getDescriptorWriter(this.field) + val reader = this.testConnection.getDescriptorReader(this.field) + val random = SplittableRandom() + + /* Generate and store test data. */ + val descriptors = this.initialize(writer, random) + val d = descriptors[random.nextInt(0, descriptors.size)] + + /* Prepare and execute query. */ + val query = SimpleBooleanQuery( + d.path, + ComparisonOperator.EQ, + "path" + ) + + /* Check results. */ + val result = reader.query(query).toList() + Assertions.assertTrue(result.isNotEmpty()) + for (r in result) { + Assertions.assertEquals(d.path, r.path) + } + } + + /** + * Tests for LIKE comparison. + */ + @Test + fun testBooleanQueryLike() { + val writer = this.testConnection.getDescriptorWriter(this.field) + val reader = this.testConnection.getDescriptorReader(this.field) + val random = SplittableRandom() + + /* Generate and store test data. */ + this.initialize(writer, random) + + /* Prepare and execute query. */ + val query = SimpleBooleanQuery( + Value.String("%.jpg"), + ComparisonOperator.LIKE, + "path" + ) + + /* Check results. */ + val result = reader.query(query).toList() + for (r in result) { + Assertions.assertTrue(r.path.value.endsWith(".jpg")) + } + } + + /** + * Tests for greater-than comparison. + */ + @Test + fun testBooleanQueryGreater() { + val writer = this.testConnection.getDescriptorWriter(this.field) + val reader = this.testConnection.getDescriptorReader(this.field) + val random = SplittableRandom() + + /* Generate and store test data. */ + this.initialize(writer, random) + + /* Prepare and execute query. */ + val size = Value.Long(random.nextLong(0, 100_000_000L)) + val query = SimpleBooleanQuery( + size, + ComparisonOperator.GR, + "size" + ) + + /* Check results. */ + val result = reader.query(query).toList() + for (r in result) { + Assertions.assertTrue(r.size.value > size.value) + } + } + + /** + * Tests for less-than comparison. + */ + @Test + fun testBooleanQueryLess() { + val writer = this.testConnection.getDescriptorWriter(this.field) + val reader = this.testConnection.getDescriptorReader(this.field) + val random = SplittableRandom() + + /* Generate and store test data. */ + this.initialize(writer, random) + + /* Prepare and execute query. */ + val size = Value.Long(random.nextLong(0, 100_000_000L)) + val query = SimpleBooleanQuery( + size, + ComparisonOperator.LE, + "size" + ) + + /* Check results. */ + val result = reader.query(query).toList() + for (r in result) { + Assertions.assertTrue(r.size.value < size.value) + } + } + + /** + * Tests for less-than comparison. + */ + @Test + fun testFulltextQuery() { + val writer = this.testConnection.getDescriptorWriter(this.field) + val reader = this.testConnection.getDescriptorReader(this.field) + val random = SplittableRandom() + + /* Generate and store test data. */ + this.initialize(writer, random) + + /* Prepare and execute query. */ + val query = SimpleFulltextQuery( + Value.String("var"), + "path" + ) + + /* Check results. */ + val result = reader.query(query).toList() + for (r in result) { + Assertions.assertTrue(r.path.value.contains("var")) + } + } + + /** + * Initializes the test data. + */ + private fun initialize(writer: DescriptorWriter, random: SplittableRandom): List { + val size = random.nextInt(500, 5000) + + /* Generate and store test data. */ + val retrievables = (0 until size).map { + Ingested(UUID.randomUUID(), "SOURCE:TEST", true) + } + Assertions.assertTrue(this.testConnection.getRetrievableWriter().addAll(retrievables)) + + val descriptors = retrievables.map { + FileSourceMetadataDescriptor(UUID.randomUUID(), it.id, this.generateRandomPath(random), Value.Long(random.nextLong(0, 100_000_000L)), this.field) + } + Assertions.assertTrue(writer.addAll(descriptors)) + return descriptors + } + + /** + * Generates a random path and wraps it in a Value.String. + * + * @return A Value.String representing a random path. + */ + private fun generateRandomPath(random: SplittableRandom): Value.String { + /* Define a list of possible root directories. */ + val root = ROOTS[random.nextInt(ROOTS.size)] + val fileName = FILENAMES[random.nextInt(FILENAMES.size)] + + /* Define a list of possible root directories. */ + val path = Paths.get(root, fileName).toString() + + /* Define a list of possible root directories. */ + return Value.String(path) + } + + + /** + * Cleans up the database after each test. + */ + @BeforeEach + open fun prepare() { + this.testConnection.getRetrievableInitializer().initialize() + this.testConnection.getDescriptorInitializer(this.field).initialize() + } + + /** + * Cleans up the database after each test. + * + */ + @AfterEach + open fun cleanup() { + this.testConnection.getRetrievableInitializer().deinitialize() + this.testConnection.getDescriptorInitializer(this.field).deinitialize() + } +} \ No newline at end of file diff --git a/vitrivr-engine-core/src/testFixtures/kotlin/org/vitrivr/engine/core/database/descriptor/vector/AbstractFloatVectorDescriptorReaderTest.kt b/vitrivr-engine-core/src/testFixtures/kotlin/org/vitrivr/engine/core/database/descriptor/vector/AbstractFloatVectorDescriptorReaderTest.kt new file mode 100644 index 00000000..9b80665e --- /dev/null +++ b/vitrivr-engine-core/src/testFixtures/kotlin/org/vitrivr/engine/core/database/descriptor/vector/AbstractFloatVectorDescriptorReaderTest.kt @@ -0,0 +1,162 @@ +package org.vitrivr.engine.core.database.descriptor.vector + +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.EnumSource +import org.vitrivr.engine.core.database.AbstractDatabaseTest +import org.vitrivr.engine.core.database.descriptor.DescriptorWriter +import org.vitrivr.engine.core.model.descriptor.Descriptor +import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.query.basics.Distance +import org.vitrivr.engine.core.model.query.basics.SortOrder +import org.vitrivr.engine.core.model.query.proximity.ProximityQuery +import org.vitrivr.engine.core.model.retrievable.Ingested +import org.vitrivr.engine.core.model.retrievable.attributes.DistanceAttribute +import org.vitrivr.engine.core.model.retrievable.attributes.ScoreAttribute +import org.vitrivr.engine.core.model.types.Value +import java.util.* + +/** + * A series of test cases the test the functionality of the [VectorDescriptorReader]. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +abstract class AbstractFloatVectorDescriptorReaderTest(schemaPath: String) : AbstractDatabaseTest(schemaPath) { + + /** The [Schema.Field] used for this [DescriptorInitializerTest]. */ + private val field: Schema.Field<*, FloatVectorDescriptor> = this.testSchema["averagecolor"]!! as Schema.Field<*, FloatVectorDescriptor> + + /** + * Tests [VectorDescriptorReader.queryAndJoin] method. + */ + @ParameterizedTest + @EnumSource(mode = EnumSource.Mode.EXCLUDE, names = ["JACCARD", "HAMMING"]) + fun testQueryAndJoin(distance: Distance) { + val writer = this.testConnection.getDescriptorWriter(this.field) + val reader = this.testConnection.getDescriptorReader(this.field) + val random = SplittableRandom() + + /* Generate and store test data. */ + val descriptors = this.initialize(writer, random) + + /* Perform nearest neighbour search. */ + val query = ProximityQuery( + Value.FloatVector(FloatArray(3) { random.nextFloat() }), + distance, + SortOrder.ASC, + 100, + fetchVector = true + ) + val result = reader.queryAndJoin(query).toList() + + /* Make manual query and compare. */ + val manual = descriptors.sortedBy { distance(it.vector, query.value) }.take(100) + result.zip(manual).forEach { + Assertions.assertEquals(it.first.id, it.second.retrievableId) + Assertions.assertTrue(it.first.hasAttribute(DistanceAttribute::class.java)) + } + } + + /** + * Tests nearest neighbour search through the [VectorDescriptorReader.query] method. + */ + @ParameterizedTest + @EnumSource(mode = EnumSource.Mode.EXCLUDE, names = ["JACCARD", "HAMMING"]) + fun testNearestNeighbourSearch(distance: Distance) { + val writer = this.testConnection.getDescriptorWriter(this.field) + val reader = this.testConnection.getDescriptorReader(this.field) + val random = SplittableRandom() + + /* Generate and store test data. */ + val descriptors = this.initialize(writer, random) + + /* Perform nearest neighbour search. */ + val query = ProximityQuery( + Value.FloatVector(FloatArray(3) { random.nextFloat() }), + distance, + SortOrder.ASC, + 100, + fetchVector = true + ) + val result = reader.query(query).toList() + + /* Make manual query and compare. */ + val manual = descriptors.sortedBy { distance(it.vector, query.value) }.take(100) + result.zip(manual).forEach { + Assertions.assertEquals(it.first.id, it.second.id) + } + } + + /** + * Tests farthest neighbour search through the [VectorDescriptorReader.query] method. + */ + @ParameterizedTest + @EnumSource(mode = EnumSource.Mode.EXCLUDE, names = ["JACCARD", "HAMMING"]) + fun testFarthestNeighbourSearch(distance: Distance) { + val writer = this.testConnection.getDescriptorWriter(this.field) + val reader = this.testConnection.getDescriptorReader(this.field) + val random = SplittableRandom() + + /* Generate and store test data. */ + val descriptors = this.initialize(writer, random) + + /* Perform nearest neighbour search. */ + val query = ProximityQuery( + Value.FloatVector(FloatArray(3) { random.nextFloat() }), + distance, + SortOrder.DESC, + 100, + fetchVector = true + ) + val result = reader.query(query).toList() + + /* Make manual query and compare. */ + val manual = descriptors.sortedByDescending { distance(it.vector, query.value) }.take(100) + result.zip(manual).forEach { + Assertions.assertEquals(it.first.id, it.second.id) + } + } + + /** + * Initializes the test data. + */ + private fun initialize(writer: DescriptorWriter, random: SplittableRandom): List { + val size = random.nextInt(500, 5000) + + /* Generate and store test data. */ + val retrievables = (0 until size).map { + Ingested(UUID.randomUUID(), "SOURCE:TEST", true) + } + Assertions.assertTrue(this.testConnection.getRetrievableWriter().addAll(retrievables)) + + val descriptors = retrievables.map { + FloatVectorDescriptor(UUID.randomUUID(), it.id, Value.FloatVector(FloatArray(3) { random.nextFloat() })) + } + Assertions.assertTrue(writer.addAll(descriptors)) + return descriptors + } + + /** + * Cleans up the database after each test. + */ + @BeforeEach + open fun prepare() { + this.testConnection.getRetrievableInitializer().initialize() + this.testConnection.getDescriptorInitializer(this.field).initialize() + } + + /** + * Cleans up the database after each test. + * + */ + @AfterEach + open fun cleanup() { + this.testConnection.getRetrievableInitializer().deinitialize() + this.testConnection.getDescriptorInitializer(this.field).deinitialize() + } +} \ No newline at end of file diff --git a/vitrivr-engine-core/src/testFixtures/kotlin/org/vitrivr/engine/core/database/retrievable/AbstractRetrievableInitializerTest.kt b/vitrivr-engine-core/src/testFixtures/kotlin/org/vitrivr/engine/core/database/retrievable/AbstractRetrievableInitializerTest.kt new file mode 100644 index 00000000..6c11b001 --- /dev/null +++ b/vitrivr-engine-core/src/testFixtures/kotlin/org/vitrivr/engine/core/database/retrievable/AbstractRetrievableInitializerTest.kt @@ -0,0 +1,66 @@ +package org.vitrivr.engine.core.database.retrievable + +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test +import org.vitrivr.engine.core.database.AbstractDatabaseTest + +/** + * An abstract set of test cases to test the proper functioning of [RetrievableInitializer] implementations. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +abstract class AbstractRetrievableInitializerTest(schemaPath: String) : AbstractDatabaseTest(schemaPath) { + /** + * Tests if the [RetrievableInitializer.isInitialized] works as expected., + */ + @Test + open fun testIsInitializedWithoutInitialization() { + Assertions.assertFalse(this.testConnection.getRetrievableInitializer().isInitialized()) + } + + /** + * Tests if the [RetrievableInitializer] can be initialized without throwing an exception. Furthermore, + * the test should check if the necessary tables have been created. + */ + @Test + open fun testInitializeEntities() { + /* Check initialization status (should be false). */ + Assertions.assertFalse(this.testConnection.getRetrievableInitializer().isInitialized()) + + /* Initialize basic tables. */ + this.testConnection.getRetrievableInitializer().initialize() + + /* Check initialization status (should be true). */ + Assertions.assertTrue(this.testConnection.getRetrievableInitializer().isInitialized()) + } + + /** + * Tests if the [RetrievableInitializer] can be initialized without throwing an exception. Furthermore, + * the test should check if the necessary tables have been created. + */ + @Test + open fun testDeInitializeEntities() { + /* Check initialization status (should be false). */ + Assertions.assertFalse(this.testConnection.getRetrievableInitializer().isInitialized()) + + /* Initialize basic tables. */ + this.testConnection.getRetrievableInitializer().initialize() + + /* Check initialization status (should be true). */ + Assertions.assertTrue(this.testConnection.getRetrievableInitializer().isInitialized()) + + this.testConnection.getRetrievableInitializer().deinitialize() + Assertions.assertFalse(this.testConnection.getRetrievableInitializer().isInitialized()) + } + + /** + * Cleans up the database after each test. + * + */ + @AfterEach + open fun cleanup() { + this.testSchema.connection.getRetrievableInitializer().deinitialize() + } +} \ No newline at end of file diff --git a/vitrivr-engine-core/src/testFixtures/kotlin/org/vitrivr/engine/core/database/retrievable/AbstractRetrievableWriterTest.kt b/vitrivr-engine-core/src/testFixtures/kotlin/org/vitrivr/engine/core/database/retrievable/AbstractRetrievableWriterTest.kt new file mode 100644 index 00000000..ee153526 --- /dev/null +++ b/vitrivr-engine-core/src/testFixtures/kotlin/org/vitrivr/engine/core/database/retrievable/AbstractRetrievableWriterTest.kt @@ -0,0 +1,154 @@ +package org.vitrivr.engine.core.database.retrievable + +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.vitrivr.engine.core.database.AbstractDatabaseTest +import org.vitrivr.engine.core.model.retrievable.Ingested +import java.util.* + +/** + * An abstract set of test cases to test the proper functioning of [RetrievableWriter] implementations. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +abstract class AbstractRetrievableWriterTest(schemaPath: String) : AbstractDatabaseTest(schemaPath) { + /** + * Tests if the [RetrievableWriter.add] works as expected. + */ + @Test + fun testAdd() { + val writer = this.testConnection.getRetrievableWriter() + val reader = this.testConnection.getRetrievableReader() + + /* Create and add retrievable. */ + val id = UUID.randomUUID() + Assertions.assertTrue(writer.add(Ingested(id, "INGESTED:TEST", false))) + + /* Check if retrievable can be read. */ + Assertions.assertEquals(1L, reader.count()) + Assertions.assertEquals("INGESTED:TEST", reader[id]?.type) + } + + /** + * Tests if the [RetrievableWriter.add] works as expected. + */ + @Test + fun testAddAll() { + val writer = this.testConnection.getRetrievableWriter() + val reader = this.testConnection.getRetrievableReader() + val size = Random().nextInt(500, 5000) + + /* Create and add retrievable. */ + val ids = (0 until size).map { UUID.randomUUID() } + val ingested = ids.map { Ingested(it, "INGESTED:TEST", false) } + Assertions.assertTrue(writer.addAll(ingested)) + + /* Check if retrievable can be read. */ + Assertions.assertEquals(ids.size.toLong(), reader.count()) + reader.getAll(ids).forEachIndexed { i, it -> + Assertions.assertTrue(ids.contains(it.id)) + Assertions.assertEquals("INGESTED:TEST", it.type) + } + } + + /** + * Tests if the [RetrievableWriter.add] works as expected. + */ + @Test + fun testDelete() { + val writer = this.testConnection.getRetrievableWriter() + val reader = this.testConnection.getRetrievableReader() + val size = Random().nextInt(500, 5000) + + /* Create and add retrievable. */ + val ids = (0 until size).map { UUID.randomUUID() } + val ingested = ids.map { Ingested(it, "INGESTED:TEST", false) } + val delete = ingested[Random().nextInt(0, ingested.size)] + + /* Execute actions. */ + Assertions.assertFalse(writer.delete(delete)) + Assertions.assertTrue(writer.addAll(ingested)) + Assertions.assertTrue(writer.delete(delete)) + + /* Check if retrievable can be read. */ + Assertions.assertEquals(ids.size.toLong() - 1L, reader.count()) + reader.getAll(ids).forEachIndexed() { i, it -> + Assertions.assertNotEquals(delete.id, it.id) + Assertions.assertEquals("INGESTED:TEST", it.type) + } + } + + /** + * Tests if the [RetrievableWriter.add] works as expected. + */ + @Test + fun testDeleteAll() { + val writer = this.testConnection.getRetrievableWriter() + val reader = this.testConnection.getRetrievableReader() + val size = Random().nextInt(500, 5000) + + /* Create and add retrievable. */ + val ids = (0 until size).map { UUID.randomUUID() } + val ingested = ids.map { Ingested(it, "INGESTED:TEST", false) } + + /* Execute actions. */ + Assertions.assertTrue(writer.addAll(ingested)) + + /* Check if retrievable can be read. */ + Assertions.assertEquals(ids.size.toLong(), reader.count()) + reader.getAll(ids).forEachIndexed() { i, it -> + Assertions.assertEquals("INGESTED:TEST", it.type) + } + + /* Execute actions. */ + Assertions.assertTrue(writer.deleteAll(ingested)) + Assertions.assertEquals(0L, reader.count()) + } + + + /** + * Tests if the [RetrievableWriter.add] works as expected. + */ + @Test + fun testUpdate() { + val writer = this.testConnection.getRetrievableWriter() + val reader = this.testConnection.getRetrievableReader() + val size = Random().nextInt(500, 5000) + + /* Create and add retrievable. */ + val ids = (0 until size).map { UUID.randomUUID() } + val ingested = ids.map { Ingested(it, "INGESTED:TEST", false) } + val update = ingested[Random().nextInt(0, ingested.size)] + + /* Execute actions. */ + Assertions.assertTrue(writer.addAll(ingested)) + + /* Check if retrievable can be read. */ + Assertions.assertEquals(ids.size.toLong(), reader.count()) + Assertions.assertTrue(writer.update(update.copy(type = "INGESTED:TEST2"))) + + /* Execute actions. */ + Assertions.assertEquals(ids.size.toLong(), reader.count()) + Assertions.assertEquals("INGESTED:TEST2", reader[update.id]?.type) + } + + /** + * Cleans up the database after each test. + */ + @BeforeEach + open fun prepare() { + this.testSchema.connection.getRetrievableInitializer().initialize() + } + + /** + * Cleans up the database after each test. + * + */ + @AfterEach + open fun cleanup() { + this.testSchema.connection.getRetrievableInitializer().deinitialize() + } +} \ No newline at end of file diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/AverageImageContentAggregator.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/AverageImageContentAggregator.kt index 12b0be36..72d0d09f 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/AverageImageContentAggregator.kt +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/AverageImageContentAggregator.kt @@ -56,7 +56,7 @@ class AverageImageContentAggregator : TransformerFactory { images.forEach { imageContent -> require(imageContent.height == height && imageContent.width == width) { "Unable to aggregate images! All images must have same dimension." } imageContent.content.getRGBArray().forEachIndexed { index, color -> - colors[index] += RGBByteColorContainer.fromRGB(color) + colors[index] += RGBByteColorContainer(color) } } diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/RepresentativeImageContentAggregator.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/RepresentativeImageContentAggregator.kt index 04bf7cb6..e91c9ccf 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/RepresentativeImageContentAggregator.kt +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/aggregators/image/RepresentativeImageContentAggregator.kt @@ -56,7 +56,7 @@ class RepresentativeImageContentAggregator : TransformerFactory { images.forEach { imageContent -> require(imageContent.height == height && imageContent.width == width) { "Unable to aggregate images! All images must have same dimension." } imageContent.content.getRGBArray().forEachIndexed { index, color -> - colors[index] += RGBByteColorContainer.fromRGB(color) + colors[index] += RGBByteColorContainer(color) } } @@ -68,7 +68,7 @@ class RepresentativeImageContentAggregator : TransformerFactory { val mostRepresentative = images.minBy { imageContent -> imageContent.content.getRGBArray().mapIndexed { index, color -> - RGBByteColorContainer.fromRGB(color).toFloatContainer().distanceTo(colors[index]) + RGBByteColorContainer(color).toFloatContainer().distanceTo(colors[index]) }.sum() } diff --git a/vitrivr-engine-module-cottontaildb/build.gradle b/vitrivr-engine-module-cottontaildb/build.gradle index 32b8331a..da3bdd7c 100644 --- a/vitrivr-engine-module-cottontaildb/build.gradle +++ b/vitrivr-engine-module-cottontaildb/build.gradle @@ -16,6 +16,9 @@ dependencies { /** gRPC and Protobuf. */ implementation group: 'io.grpc', name: 'grpc-all', version: version_grpc implementation group: 'com.google.protobuf', name: 'protobuf-java', version: version_protobuf + + /** vitrivr engine Core is required for running tests. */ + testImplementation(testFixtures(project(':vitrivr-engine-core'))) } /* Publication of vitrivr engine query to Maven Central. */ diff --git a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/Common.kt b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/Common.kt index 356a6f73..310ffae0 100644 --- a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/Common.kt +++ b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/Common.kt @@ -3,7 +3,7 @@ package org.vitrivr.engine.plugin.cottontaildb import org.vitrivr.cottontail.client.language.basics.predicate.Compare import org.vitrivr.cottontail.core.types.Types import org.vitrivr.cottontail.core.values.* -import org.vitrivr.engine.core.model.descriptor.FieldSchema +import org.vitrivr.engine.core.model.descriptor.Attribute import org.vitrivr.engine.core.model.descriptor.scalar.* import org.vitrivr.engine.core.model.descriptor.vector.* import org.vitrivr.engine.core.model.query.basics.ComparisonOperator @@ -39,9 +39,6 @@ const val DESCRIPTOR_ENTITY_PREFIX = "descriptor" /** The column name of a descriptor ID. */ const val DESCRIPTOR_ID_COLUMN_NAME = "descriptorId" -/** The column name of a descriptor ID. */ -const val DESCRIPTOR_COLUMN_NAME = "descriptor" - /** The column name used to describe a distance.*/ const val DISTANCE_COLUMN_NAME = "distance" @@ -49,48 +46,26 @@ const val DISTANCE_COLUMN_NAME = "distance" const val SCORE_COLUMN_NAME = "score" /** - * Converts a vitrivr-engine [FieldSchema] to a Cottontail DB [Types]. + * Converts a vitrivr-engine [Attribute] to a Cottontail DB [Types]. * * @return [Compare.Operator] used for this [SimpleBooleanQuery] */ -internal fun FieldSchema.toCottontailType(): Types<*> { - val vector = this.dimensions.size == 1 - return when (this.type) { - Type.STRING -> Types.String - Type.BYTE -> Types.Byte - Type.SHORT -> Types.Short - Type.BOOLEAN -> if (vector) { - Types.BooleanVector(this.dimensions[0]) - } else { - Types.Boolean - } - - Type.INT -> if (vector) { - Types.IntVector(this.dimensions[0]) - } else { - Types.Int - } - - Type.LONG -> if (vector) { - Types.LongVector(this.dimensions[0]) - } else { - Types.Long - } - - Type.FLOAT -> if (vector) { - Types.FloatVector(this.dimensions[0]) - } else { - Types.Float - } - - Type.DOUBLE -> if (vector) { - Types.DoubleVector(this.dimensions[0]) - } else { - Types.Double - } - - Type.DATETIME -> Types.Date - } +internal fun Type.toCottontailType(): Types<*> = when (this) { + Type.String -> Types.String + Type.Text -> Types.String + Type.Byte -> Types.Byte + Type.Short -> Types.Short + Type.Boolean -> Types.Boolean + Type.Int -> Types.Int + Type.Long -> Types.Long + Type.Float -> Types.Float + Type.Double -> Types.Double + Type.Datetime -> Types.Date + is Type.BooleanVector -> Types.BooleanVector(this.dimensions) + is Type.DoubleVector -> Types.DoubleVector(this.dimensions) + is Type.FloatVector -> Types.FloatVector(this.dimensions) + is Type.IntVector -> Types.IntVector(this.dimensions) + is Type.LongVector -> Types.LongVector(this.dimensions) } /** @@ -150,7 +125,13 @@ internal fun Value<*>.toCottontailValue(): PublicValue = when (this) { is Value.Long -> LongValue(this.value) is Value.Short -> ShortValue(this.value) is Value.String -> StringValue(this.value) + is Value.Text -> StringValue(this.value) is Value.DateTime -> DateValue(this.value) + is Value.BooleanVector -> BooleanVectorValue(this.value) + is Value.DoubleVector -> DoubleVectorValue(this.value) + is Value.FloatVector -> FloatVectorValue(this.value) + is Value.IntVector -> IntVectorValue(this.value) + is Value.LongVector -> LongVectorValue(this.value) } /** diff --git a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/CottontailConnection.kt b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/CottontailConnection.kt index ba533b71..76eeab3c 100644 --- a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/CottontailConnection.kt +++ b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/CottontailConnection.kt @@ -1,13 +1,20 @@ package org.vitrivr.engine.plugin.cottontaildb +import io.github.oshai.kotlinlogging.KLogger +import io.github.oshai.kotlinlogging.KotlinLogging.logger import io.grpc.ManagedChannel import io.grpc.ManagedChannelBuilder +import io.grpc.StatusRuntimeException import org.vitrivr.cottontail.client.SimpleClient +import org.vitrivr.cottontail.client.language.ddl.CreateSchema import org.vitrivr.engine.core.database.AbstractConnection import org.vitrivr.engine.plugin.cottontaildb.retrievable.RetrievableInitializer import org.vitrivr.engine.plugin.cottontaildb.retrievable.RetrievableReader import org.vitrivr.engine.plugin.cottontaildb.retrievable.RetrievableWriter +/** Defines [KLogger] of the class. */ +internal val LOGGER: KLogger = logger("org.vitrivr.engine.plugin.cottontaildb.CottontailConnection") + /** * A [AbstractConnection] to connect to a Cottontail DB instance. * @@ -22,6 +29,34 @@ class CottontailConnection(provider: CottontailConnectionProvider, schemaName: S /** The [SimpleClient] instance used by this [CottontailConnection]. */ internal val client = SimpleClient(this.channel) + + init { + try { + this.client.create(CreateSchema(this.schemaName).ifNotExists()) + } catch (e: StatusRuntimeException) { + LOGGER.error(e) { "Failed to create schema '${this.schemaName}' due to exception." } + } + } + + /** + * Tries to execute a given action within a database transaction. + * + * @param action The action to execute within the transaction. + */ + @Synchronized + override fun withTransaction(action: (Unit) -> T): T { + val transactionId = this.client.begin() + try { + val ret = action.invoke(Unit) + this.client.commit(transactionId) + return ret + } catch (e: Throwable) { + this.client.rollback(transactionId) + LOGGER.error(e) { "Failed to execute action in transaction due to erro.." } + throw e + } + } + /** * Generates and returns a [RetrievableInitializer] for this [CottontailConnection]. * diff --git a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/CottontailConnectionProvider.kt b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/CottontailConnectionProvider.kt index 68719230..44625cb4 100644 --- a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/CottontailConnectionProvider.kt +++ b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/CottontailConnectionProvider.kt @@ -1,33 +1,29 @@ package org.vitrivr.engine.plugin.cottontaildb +import org.vitrivr.engine.core.database.AbstractConnectionProvider import org.vitrivr.engine.core.database.Connection import org.vitrivr.engine.core.database.ConnectionProvider import org.vitrivr.engine.core.database.descriptor.DescriptorProvider -import org.vitrivr.engine.core.model.descriptor.Descriptor import org.vitrivr.engine.core.model.descriptor.scalar.* import org.vitrivr.engine.core.model.descriptor.struct.LabelDescriptor -import org.vitrivr.engine.core.model.descriptor.struct.RasterDescriptor -import org.vitrivr.engine.core.model.descriptor.struct.SkeletonDescriptor import org.vitrivr.engine.core.model.descriptor.struct.metadata.MediaDimensionsDescriptor import org.vitrivr.engine.core.model.descriptor.struct.metadata.Rectangle2DMetadataDescriptor import org.vitrivr.engine.core.model.descriptor.struct.metadata.TemporalMetadataDescriptor import org.vitrivr.engine.core.model.descriptor.struct.metadata.source.FileSourceMetadataDescriptor -import org.vitrivr.engine.core.model.descriptor.struct.metadata.source.MapStructDescriptor import org.vitrivr.engine.core.model.descriptor.struct.metadata.source.VideoSourceMetadataDescriptor import org.vitrivr.engine.core.model.descriptor.vector.* import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.plugin.cottontaildb.descriptors.scalar.ScalarDescriptorProvider -import org.vitrivr.engine.plugin.cottontaildb.descriptors.string.StringDescriptorProvider -import java.util.* -import kotlin.reflect.KClass +import org.vitrivr.engine.plugin.cottontaildb.descriptors.struct.StructDescriptorProvider +import org.vitrivr.engine.plugin.cottontaildb.descriptors.vector.VectorDescriptorProvider /** * Implementation of the [ConnectionProvider] interface for Cottontail DB. * * @author Ralph Gasser - * @version 1.0.0 + * @version 1.1.0 */ -class CottontailConnectionProvider: ConnectionProvider { +class CottontailConnectionProvider: AbstractConnectionProvider() { companion object { /** Name of the host parameter. */ @@ -49,36 +45,33 @@ class CottontailConnectionProvider: ConnectionProvider { /** The version of the [CottontailConnectionProvider]. */ override val version: String = "1.0.0" - /** List of registered [LinkedList]*/ - private val registered = mutableMapOf, DescriptorProvider<*>>( + /** + * This method is called during initialization of the [CottontailConnectionProvider] and can be used to register [DescriptorProvider]s. + */ + override fun initialize() { /* Scalar descriptors. */ - BooleanDescriptor::class to ScalarDescriptorProvider, - IntDescriptor::class to ScalarDescriptorProvider, - LongDescriptor::class to ScalarDescriptorProvider, - FloatDescriptor::class to ScalarDescriptorProvider, - DoubleDescriptor::class to ScalarDescriptorProvider, - - /* String descriptor. */ - StringDescriptor::class to StringDescriptorProvider, + this.register(BooleanDescriptor::class, ScalarDescriptorProvider) + this.register(IntDescriptor::class, ScalarDescriptorProvider) + this.register(LongDescriptor::class, ScalarDescriptorProvider) + this.register(FloatDescriptor::class, ScalarDescriptorProvider) + this.register(DoubleDescriptor::class, ScalarDescriptorProvider) + this.register(StringDescriptor::class, ScalarDescriptorProvider) /* Vector descriptors. */ - BooleanVectorDescriptor::class to org.vitrivr.engine.plugin.cottontaildb.descriptors.vector.VectorDescriptorProvider, - IntVectorDescriptor::class to org.vitrivr.engine.plugin.cottontaildb.descriptors.vector.VectorDescriptorProvider, - LongVectorDescriptor::class to org.vitrivr.engine.plugin.cottontaildb.descriptors.vector.VectorDescriptorProvider, - FloatVectorDescriptor::class to org.vitrivr.engine.plugin.cottontaildb.descriptors.vector.VectorDescriptorProvider, - DoubleVectorDescriptor::class to org.vitrivr.engine.plugin.cottontaildb.descriptors.vector.VectorDescriptorProvider, + this.register(BooleanVectorDescriptor::class, VectorDescriptorProvider) + this.register(IntVectorDescriptor::class, VectorDescriptorProvider) + this.register(LongVectorDescriptor::class, VectorDescriptorProvider) + this.register(FloatVectorDescriptor::class, VectorDescriptorProvider) + this.register(DoubleVectorDescriptor::class, VectorDescriptorProvider) /* Struct descriptor. */ - LabelDescriptor::class to org.vitrivr.engine.plugin.cottontaildb.descriptors.struct.StructDescriptorProvider, - FileSourceMetadataDescriptor::class to org.vitrivr.engine.plugin.cottontaildb.descriptors.struct.StructDescriptorProvider, - VideoSourceMetadataDescriptor::class to org.vitrivr.engine.plugin.cottontaildb.descriptors.struct.StructDescriptorProvider, - TemporalMetadataDescriptor::class to org.vitrivr.engine.plugin.cottontaildb.descriptors.struct.StructDescriptorProvider, - Rectangle2DMetadataDescriptor::class to org.vitrivr.engine.plugin.cottontaildb.descriptors.struct.StructDescriptorProvider, - MediaDimensionsDescriptor::class to org.vitrivr.engine.plugin.cottontaildb.descriptors.struct.StructDescriptorProvider, - SkeletonDescriptor::class to org.vitrivr.engine.plugin.cottontaildb.descriptors.struct.StructDescriptorProvider, - RasterDescriptor::class to org.vitrivr.engine.plugin.cottontaildb.descriptors.struct.StructDescriptorProvider, - MapStructDescriptor::class to org.vitrivr.engine.plugin.cottontaildb.descriptors.struct.StructDescriptorProvider - ) + this.register(LabelDescriptor::class, StructDescriptorProvider) + this.register(FileSourceMetadataDescriptor::class, StructDescriptorProvider) + this.register(VideoSourceMetadataDescriptor::class, StructDescriptorProvider) + this.register(TemporalMetadataDescriptor::class, StructDescriptorProvider) + this.register(Rectangle2DMetadataDescriptor::class, StructDescriptorProvider) + this.register(MediaDimensionsDescriptor::class, StructDescriptorProvider) + } /** * Opens a new [CottontailConnection] for the given [Schema]. @@ -92,25 +85,4 @@ class CottontailConnectionProvider: ConnectionProvider { parameters[PARAMETER_NAME_HOST] ?: PARAMETER_DEFAULT_HOST, parameters[PARAMETER_NAME_PORT]?.toIntOrNull() ?: PARAMETER_DEFAULT_PORT ) - - /** - * Registers an [DescriptorProvider] for a particular [KClass] of [Descriptor] with this [Connection]. - * - * This method is an extension point to add support for new [Descriptor]s to a pre-existing database driver. - * - * @param descriptorClass The [KClass] of the [Descriptor] to register [DescriptorProvider] for. - * @param provider The [DescriptorProvider] to register. - */ - override fun register(descriptorClass: KClass, provider: DescriptorProvider<*>) { - this.registered[descriptorClass] = provider - } - - /** - * Obtains an [DescriptorProvider] for a particular [KClass] of [Descriptor], that has been registered with this [ConnectionProvider]. - * - * @param descriptorClass The [KClass] of the [Descriptor] to lookup [DescriptorProvider] for. - * @return The registered [DescriptorProvider] . - */ - @Suppress("UNCHECKED_CAST") - override fun obtain(descriptorClass: KClass): DescriptorProvider = this.registered[descriptorClass] as DescriptorProvider } \ No newline at end of file diff --git a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/AbstractDescriptorInitializer.kt b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/AbstractDescriptorInitializer.kt deleted file mode 100644 index c0d6ed83..00000000 --- a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/AbstractDescriptorInitializer.kt +++ /dev/null @@ -1,49 +0,0 @@ -package org.vitrivr.engine.plugin.cottontaildb.descriptors - -import io.grpc.StatusRuntimeException -import org.vitrivr.cottontail.client.language.ddl.ListEntities -import org.vitrivr.cottontail.client.language.ddl.TruncateEntity -import org.vitrivr.cottontail.core.database.Name -import org.vitrivr.engine.core.database.descriptor.DescriptorInitializer -import org.vitrivr.engine.core.model.descriptor.Descriptor -import org.vitrivr.engine.core.model.metamodel.Schema -import org.vitrivr.engine.plugin.cottontaildb.CottontailConnection -import org.vitrivr.engine.plugin.cottontaildb.DESCRIPTOR_ENTITY_PREFIX - -/** - * An abstract implementation of a [DescriptorInitializer] for Cottontail DB. - * - * @author Ralph Gasser - * @version 1.0.0 - */ -abstract class AbstractDescriptorInitializer(final override val field: Schema.Field<*, D>, protected val connection: CottontailConnection) : DescriptorInitializer { - /** The [Name.EntityName] used by this [Descriptor]. */ - protected val entityName: Name.EntityName = Name.EntityName.create(this.field.schema.name, "${DESCRIPTOR_ENTITY_PREFIX}_${this.field.fieldName.lowercase()}") - - /** - * Checks if the schema for this [AbstractDescriptorInitializer] has been properly initialized. - * - * @return True if entity has been initialized, false otherwise. - */ - override fun isInitialized(): Boolean = try { - this.connection.client.list(ListEntities(this.entityName.schema)).asSequence().any { - Name.EntityName.parse(it.asString(0)!!) == this.entityName - } - } catch (e: StatusRuntimeException) { - false - } - - /** - * Truncates the entity backing this [AbstractDescriptorInitializer]. - */ - override fun truncate() { - val truncate = TruncateEntity(this.entityName) - try { - this.connection.client.truncate(truncate).use { - it.hasNext() - } - } catch (e: StatusRuntimeException) { - /* TODO: Log. */ - } - } -} \ No newline at end of file diff --git a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/AbstractDescriptorReader.kt b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/AbstractDescriptorReader.kt index b2676ba2..bf4805ef 100644 --- a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/AbstractDescriptorReader.kt +++ b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/AbstractDescriptorReader.kt @@ -1,7 +1,5 @@ package org.vitrivr.engine.plugin.cottontaildb.descriptors -import io.github.oshai.kotlinlogging.KLogger -import io.github.oshai.kotlinlogging.KotlinLogging import io.grpc.StatusRuntimeException import org.vitrivr.cottontail.client.language.basics.expression.Column import org.vitrivr.cottontail.client.language.basics.expression.Literal @@ -19,15 +17,13 @@ import org.vitrivr.engine.core.model.retrievable.Retrieved import org.vitrivr.engine.plugin.cottontaildb.* import java.util.* -private val logger: KLogger = KotlinLogging.logger {} - /** * An abstract implementation of a [DescriptorReader] for Cottontail DB. * * @author Ralph Gasser - * @version 1.0.0 + * @version 1.0.1 */ -abstract class AbstractDescriptorReader(final override val field: Schema.Field<*, D>, protected val connection: CottontailConnection) : DescriptorReader { +abstract class AbstractDescriptorReader(final override val field: Schema.Field<*, D>, override val connection: CottontailConnection) : DescriptorReader { /** The [Name.EntityName] used by this [Descriptor]. */ protected val entityName: Name.EntityName = Name.EntityName.create(this.field.schema.name, "${DESCRIPTOR_ENTITY_PREFIX}_${this.field.fieldName.lowercase()}") @@ -51,7 +47,7 @@ abstract class AbstractDescriptorReader(final override val field ret } } catch (e: StatusRuntimeException) { - logger.error(e) { "Failed to retrieve descriptor $descriptorId due to exception." } + LOGGER.error(e) { "Failed to retrieve descriptor $descriptorId due to exception." } null } } @@ -62,13 +58,13 @@ abstract class AbstractDescriptorReader(final override val field * @param retrievableId The [RetrievableId] to search for. * @return [Sequence] of [Descriptor] of type [D] */ - override fun getFor(retrievableId: RetrievableId): Sequence { + override fun getForRetrievable(retrievableId: RetrievableId): Sequence { val query = org.vitrivr.cottontail.client.language.dql.Query(this.entityName).where(Compare(Column(this.entityName.column(RETRIEVABLE_ID_COLUMN_NAME)), Compare.Operator.EQUAL, Literal(UuidValue(retrievableId)))) return try { val result = this.connection.client.query(query) result.asSequence().map { this.tupleToDescriptor(it) } } catch (e: StatusRuntimeException) { - logger.error(e) { "Failed to retrieve descriptors for retrievable $retrievableId due to exception." } + LOGGER.error(e) { "Failed to retrieve descriptors for retrievable $retrievableId due to exception." } emptySequence() } } @@ -86,7 +82,7 @@ abstract class AbstractDescriptorReader(final override val field val result = this.connection.client.query(query) result.next().asBoolean(0) ?: false } catch (e: StatusRuntimeException) { - logger.error(e) { "Failed to retrieve descriptor $descriptorId due to exception." } + LOGGER.error(e) { "Failed to retrieve descriptor $descriptorId due to exception." } false } } @@ -102,7 +98,7 @@ abstract class AbstractDescriptorReader(final override val field val result = this.connection.client.query(query) result.asSequence().map { this.tupleToDescriptor(it) } } catch (e: StatusRuntimeException) { - logger.error(e) { "Failed to retrieve descriptors due to exception." } + LOGGER.error(e) { "Failed to retrieve descriptors due to exception." } emptySequence() } } @@ -120,7 +116,7 @@ abstract class AbstractDescriptorReader(final override val field val result = this.connection.client.query(query) result.asSequence().map { this.tupleToDescriptor(it) } } catch (e: StatusRuntimeException) { - logger.error(e) { "Failed to retrieve descriptors for provided descriptor IDs due to exception." } + LOGGER.error(e) { "Failed to retrieve descriptors for provided descriptor IDs due to exception." } emptySequence() } } @@ -131,14 +127,14 @@ abstract class AbstractDescriptorReader(final override val field * @param retrievableIds A [Iterable] of [RetrievableId]s to return [Descriptor]s for * @return [Sequence] of [Descriptor] of type [D] */ - override fun getAllFor(retrievableIds: Iterable): Sequence { + override fun getAllForRetrievable(retrievableIds: Iterable): Sequence { val query = org.vitrivr.cottontail.client.language.dql.Query(this.entityName) .where(Compare(Column(this.entityName.column(RETRIEVABLE_ID_COLUMN_NAME)), Compare.Operator.IN, org.vitrivr.cottontail.client.language.basics.expression.List(retrievableIds.map { UuidValue(it) }.toTypedArray()))) return try { val result = this.connection.client.query(query) result.asSequence().map { this.tupleToDescriptor(it) } } catch (e: StatusRuntimeException) { - logger.error(e) { "Failed to retrieve descriptors for provided retrievable IDs due to exception." } + LOGGER.error(e) { "Failed to retrieve descriptors for provided retrievable IDs due to exception." } emptySequence() } } @@ -160,7 +156,7 @@ abstract class AbstractDescriptorReader(final override val field result.close() ret } catch (e: StatusRuntimeException) { - logger.error(e) { "Failed to count descriptors due to exception." } + LOGGER.error(e) { "Failed to count descriptors due to exception." } 0L } } diff --git a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/AbstractDescriptorWriter.kt b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/AbstractDescriptorWriter.kt deleted file mode 100644 index 3e7e030c..00000000 --- a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/AbstractDescriptorWriter.kt +++ /dev/null @@ -1,83 +0,0 @@ -package org.vitrivr.engine.plugin.cottontaildb.descriptors - -import io.github.oshai.kotlinlogging.KLogger -import io.github.oshai.kotlinlogging.KotlinLogging -import io.grpc.StatusRuntimeException -import org.vitrivr.cottontail.client.language.basics.expression.Column -import org.vitrivr.cottontail.client.language.basics.expression.Literal -import org.vitrivr.cottontail.client.language.basics.predicate.Compare -import org.vitrivr.cottontail.client.language.dml.Delete -import org.vitrivr.cottontail.core.database.Name -import org.vitrivr.cottontail.core.values.UuidValue -import org.vitrivr.engine.core.database.descriptor.DescriptorWriter -import org.vitrivr.engine.core.model.descriptor.Descriptor -import org.vitrivr.engine.core.model.metamodel.Schema -import org.vitrivr.engine.plugin.cottontaildb.CottontailConnection -import org.vitrivr.engine.plugin.cottontaildb.DESCRIPTOR_ENTITY_PREFIX -import org.vitrivr.engine.plugin.cottontaildb.DESCRIPTOR_ID_COLUMN_NAME - -private val logger: KLogger = KotlinLogging.logger {} - -/** - * An abstract implementation of a [DescriptorWriter] for Cottontail DB. - - * @author Ralph Gasser - * @version 1.0.0 - */ -abstract class AbstractDescriptorWriter(final override val field: Schema.Field<*, D>, protected val connection: CottontailConnection) : DescriptorWriter { - /** The [Name.EntityName] used by this [Descriptor]. */ - protected val entityName: Name.EntityName = Name.EntityName.create(this.field.schema.name, "${DESCRIPTOR_ENTITY_PREFIX}_${this.field.fieldName.lowercase()}") - - /** - * Deletes (writes) a [Descriptor] of type [D] using this [AbstractDescriptorWriter]. - * - * @param item A [Descriptor]s to delete. - * @return True on success, false otherwise. - */ - override fun delete(item: D): Boolean { - val delete = Delete(this.entityName).where( - Compare( - Column(this.entityName.column(DESCRIPTOR_ID_COLUMN_NAME)), - Compare.Operator.EQUAL, - Literal(UuidValue(item.id)) - ) - ) - - /* Delete values. */ - return try { - this.connection.client.delete(delete).use { - it.hasNext() - } - } catch (e: StatusRuntimeException) { - logger.error(e) { "Failed to delete scalar descriptor due to exception." } - false - } - } - - /** - * Deletes (writes) [Descriptor]s of type [D] using this [AbstractDescriptorWriter]. - * - * @param items A [Iterable] of [Descriptor]s to delete. - * @return True on success, false otherwise. - */ - override fun deleteAll(items: Iterable): Boolean { - val ids = items.map { UuidValue(it.id) } - val delete = Delete(this.entityName).where( - Compare( - Column(this.entityName.column(DESCRIPTOR_ID_COLUMN_NAME)), - Compare.Operator.IN, - org.vitrivr.cottontail.client.language.basics.expression.List(ids.toTypedArray()) - ) - ) - - /* Delete values. */ - return try { - this.connection.client.delete(delete).use { - it.hasNext() - } - } catch (e: StatusRuntimeException) { - logger.error(e) { "Failed to delete descriptor due to exception." } - false - } - } -} \ No newline at end of file diff --git a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/CottontailDescriptorInitializer.kt b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/CottontailDescriptorInitializer.kt new file mode 100644 index 00000000..3e8727e7 --- /dev/null +++ b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/CottontailDescriptorInitializer.kt @@ -0,0 +1,116 @@ +package org.vitrivr.engine.plugin.cottontaildb.descriptors + +import io.grpc.StatusRuntimeException +import org.vitrivr.cottontail.client.language.ddl.* +import org.vitrivr.cottontail.core.database.Name +import org.vitrivr.cottontail.core.types.Types +import org.vitrivr.cottontail.grpc.CottontailGrpc +import org.vitrivr.engine.core.config.schema.IndexType +import org.vitrivr.engine.core.database.Initializer.Companion.INDEX_TYPE_PARAMETER_NAME +import org.vitrivr.engine.core.database.descriptor.DescriptorInitializer +import org.vitrivr.engine.core.model.descriptor.Descriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.plugin.cottontaildb.* + +/** + * An abstract implementation of a [DescriptorInitializer] for Cottontail DB. + * + * @author Ralph Gasser + * @version 1.1.0 + */ +open class CottontailDescriptorInitializer(final override val field: Schema.Field<*, D>, protected val connection: CottontailConnection) : DescriptorInitializer { + /** The [Name.EntityName] used by this [Descriptor]. */ + protected val entityName: Name.EntityName = Name.EntityName.create(this.field.schema.name, "${DESCRIPTOR_ENTITY_PREFIX}_${this.field.fieldName.lowercase()}") + + /** + * Initializes the Cottontail DB entity backing this [CottontailDescriptorInitializer]. + */ + override fun initialize() { + /* Prepare query. */ + val create = CreateEntity(this.entityName) + .column(Name.ColumnName.create(DESCRIPTOR_ID_COLUMN_NAME), Types.Uuid, nullable = false, primaryKey = true, autoIncrement = false) + .column(Name.ColumnName.create(RETRIEVABLE_ID_COLUMN_NAME), Types.Uuid, nullable = false, primaryKey = false, autoIncrement = false) + + /* Append fields. */ + for (field in this.field.analyser.prototype(this.field).layout()) { + create.column(Name.ColumnName.create(field.name), field.type.toCottontailType(), nullable = field.nullable, primaryKey = false, autoIncrement = false) + } + + try { + /* Try to create entity. */ + this.connection.client.create(create) + } catch (e: StatusRuntimeException) { + LOGGER.error(e) { "Failed to initialize entity '${this.entityName}' due to exception." } + } + + /* Create indexes (optional). */ + for (index in this.field.indexes) { + require(index.attributes.size == 1) { "Cottontail DB currently only supports single-column indexes." } + try { + val createIndex = when (index.type) { + IndexType.SCALAR -> { + val type = CottontailGrpc.IndexType.valueOf(index.parameters[INDEX_TYPE_PARAMETER_NAME]?.uppercase() ?: "BTREE") + require(type in setOf(CottontailGrpc.IndexType.BTREE, CottontailGrpc.IndexType.BTREE_UQ)) { "Index type '$type' is not supported for scalar search." } + CreateIndex(this.entityName, CottontailGrpc.IndexType.BTREE).column(index.attributes.first()) + } + IndexType.FULLTEXT -> CreateIndex(this.entityName, CottontailGrpc.IndexType.LUCENE).column(index.attributes.first()) + IndexType.NNS -> { + val type = CottontailGrpc.IndexType.valueOf(index.parameters[INDEX_TYPE_PARAMETER_NAME]?.uppercase() ?: "PQ") + require(type in setOf(CottontailGrpc.IndexType.VAF, CottontailGrpc.IndexType.PQ, CottontailGrpc.IndexType.LSH, CottontailGrpc.IndexType.IVFPQ)) { "Index type '$type' is not supported for vector search." } + CreateIndex(this.entityName, type).column(index.attributes.first()).apply { + for ((k,v,) in index.parameters) { + if (k != INDEX_TYPE_PARAMETER_NAME) { + this.param(k, v) + } + } + } + } + } + this.connection.client.create(createIndex) + } catch (e: StatusRuntimeException) { + LOGGER.error(e) { "Failed to create index ${index.type} for entity '$entityName' due to exception." } + throw e + } + } + } + + /** + * Initializes the Cottontail DB entity backing this [CottontailDescriptorInitializer]. + */ + override fun deinitialize() { + try { + /* Try to drop entity. */ + this.connection.client.drop(DropEntity(this.entityName)) + } catch (e: StatusRuntimeException) { + LOGGER.error(e) { "Failed to de-initialize entity '${this.entityName}' due to exception." } + } + } + + /** + * Checks if the schema for this [CottontailDescriptorInitializer] has been properly initialized. + * + * @return True if entity has been initialized, false otherwise. + */ + override fun isInitialized(): Boolean = try { + this.connection.client.list(ListEntities(this.entityName.schema)).asSequence().any { + Name.EntityName.parse(it.asString(0)!!) == this.entityName + } + } catch (e: StatusRuntimeException) { + LOGGER.error(e) { "Failed to check initialization status of entity '${this.entityName}' due to exception." } + false + } + + /** + * Truncates the entity backing this [CottontailDescriptorInitializer]. + */ + override fun truncate() { + val truncate = TruncateEntity(this.entityName) + try { + this.connection.client.truncate(truncate).use { + it.hasNext() + } + } catch (e: StatusRuntimeException) { + LOGGER.error(e) { "Failed to truncate entity '${this.entityName}' due to exception." } + } + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/CottontailDescriptorWriter.kt b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/CottontailDescriptorWriter.kt new file mode 100644 index 00000000..f62f22a5 --- /dev/null +++ b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/CottontailDescriptorWriter.kt @@ -0,0 +1,184 @@ +package org.vitrivr.engine.plugin.cottontaildb.descriptors + +import io.grpc.StatusRuntimeException +import org.vitrivr.cottontail.client.language.basics.expression.Column +import org.vitrivr.cottontail.client.language.basics.expression.Literal +import org.vitrivr.cottontail.client.language.basics.predicate.Compare +import org.vitrivr.cottontail.client.language.dml.BatchInsert +import org.vitrivr.cottontail.client.language.dml.Delete +import org.vitrivr.cottontail.client.language.dml.Insert +import org.vitrivr.cottontail.client.language.dml.Update +import org.vitrivr.cottontail.core.database.Name +import org.vitrivr.cottontail.core.values.UuidValue +import org.vitrivr.engine.core.database.descriptor.DescriptorWriter +import org.vitrivr.engine.core.model.descriptor.Descriptor +import org.vitrivr.engine.core.model.descriptor.struct.StructDescriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.plugin.cottontaildb.* + +/** + * An abstract implementation of a [DescriptorWriter] for Cottontail DB. + * + * @author Ralph Gasser + * @version 1.1.0 + */ +open class CottontailDescriptorWriter(final override val field: Schema.Field<*, D>, override val connection: CottontailConnection) : DescriptorWriter { + /** The [Name.EntityName] used by this [Descriptor]. */ + protected val entityName: Name.EntityName = Name.EntityName.create(this.field.schema.name, "${DESCRIPTOR_ENTITY_PREFIX}_${this.field.fieldName.lowercase()}") + + /** + * Adds (writes) a single [StructDescriptor] using this [CottontailDescriptorWriter]. + * + * @param item The [StructDescriptor] to write. + * @return True on success, false otherwise. + */ + override fun add(item: D): Boolean { + val insert = Insert(this.entityName).values( + DESCRIPTOR_ID_COLUMN_NAME to UuidValue(item.id), + RETRIEVABLE_ID_COLUMN_NAME to UuidValue(item.retrievableId ?: throw IllegalArgumentException("A struct descriptor must be associated with a retrievable ID.")), + ) + + /* Append fields. */ + for ((attribute, value) in item.values()) { + insert.value(attribute, value?.toCottontailValue()) + } + + return try { + this.connection.client.insert(insert).use { + it.hasNext() + } + } catch (e: StatusRuntimeException) { + LOGGER.error(e) { "Failed to persist descriptor ${item.id} due to exception." } + false + } + } + + /** + * Adds (writes) a batch of [StructDescriptor] using this [CottontailDescriptorWriter]. + * + * @param items A [Iterable] of [StructDescriptor]s to write. + * @return True on success, false otherwise. + */ + override fun addAll(items: Iterable): Boolean { + /* Prepare insert query. */ + var index = 0 + val insert = BatchInsert(this.entityName) + + /* Insert values. */ + for (item in items) { + val attributes = item.layout() + if (index == 0) { + val columns = Array(attributes.size + 2) { + when (it) { + 0 -> DESCRIPTOR_ID_COLUMN_NAME + 1 -> RETRIEVABLE_ID_COLUMN_NAME + else -> attributes[it - 2].name + } + } + insert.columns(*columns) + } + val inserts: Array = Array(attributes.size + 2) { + when (it) { + 0 -> UuidValue(item.id) + 1 -> item.retrievableId?.let { v -> UuidValue(v) } + else -> item.values()[attributes[it - 2].name]?.toCottontailValue() + } + } + insert.any(*inserts) + index += 1 + } + + /* Insert values. */ + return try { + this.connection.client.insert(insert).use { + it.hasNext() + } + } catch (e: StatusRuntimeException) { + LOGGER.error(e) { "Failed to persist ${index + 1} scalar descriptors due to exception." } + false + } + } + + /** + * Updates a specific [Descriptor] of type [D] using this [CottontailDescriptorWriter]. + * + * @param item A [Descriptor]s to update. + * @return True on success, false otherwise. + */ + override fun update(item: D): Boolean { + val update = Update(this.entityName).where( + Compare( + Column(this.entityName.column(DESCRIPTOR_ID_COLUMN_NAME)), + Compare.Operator.EQUAL, + Literal(UuidValue(item.id)) + ) + ) + + /* Append values. */ + for ((field, value) in item.values()) { + update.any(field to value?.toCottontailValue()) + } + + /* Update values. */ + return try { + this.connection.client.update(update) + true + } catch (e: StatusRuntimeException) { + LOGGER.error(e) { "Failed to update descriptor due to exception." } + false + } + } + + /** + * Deletes (writes) a [Descriptor] of type [D] using this [CottontailDescriptorWriter]. + * + * @param item A [Descriptor]s to delete. + * @return True on success, false otherwise. + */ + override fun delete(item: D): Boolean { + val delete = Delete(this.entityName).where( + Compare( + Column(this.entityName.column(DESCRIPTOR_ID_COLUMN_NAME)), + Compare.Operator.EQUAL, + Literal(UuidValue(item.id)) + ) + ) + + /* Delete values. */ + return try { + this.connection.client.delete(delete).use { + it.hasNext() + } + } catch (e: StatusRuntimeException) { + LOGGER.error(e) { "Failed to delete scalar descriptor due to exception." } + false + } + } + + /** + * Deletes (writes) [Descriptor]s of type [D] using this [CottontailDescriptorWriter]. + * + * @param items A [Iterable] of [Descriptor]s to delete. + * @return True on success, false otherwise. + */ + override fun deleteAll(items: Iterable): Boolean { + val ids = items.map { UuidValue(it.id) } + val delete = Delete(this.entityName).where( + Compare( + Column(this.entityName.column(DESCRIPTOR_ID_COLUMN_NAME)), + Compare.Operator.IN, + org.vitrivr.cottontail.client.language.basics.expression.List(ids.toTypedArray()) + ) + ) + + /* Delete values. */ + return try { + this.connection.client.delete(delete).use { + it.hasNext() + } + } catch (e: StatusRuntimeException) { + LOGGER.error(e) { "Failed to delete descriptor due to exception." } + false + } + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/scalar/ScalarDescriptorInitializer.kt b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/scalar/ScalarDescriptorInitializer.kt deleted file mode 100644 index b0591df1..00000000 --- a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/scalar/ScalarDescriptorInitializer.kt +++ /dev/null @@ -1,43 +0,0 @@ -package org.vitrivr.engine.plugin.cottontaildb.descriptors.scalar - -import io.github.oshai.kotlinlogging.KLogger -import io.github.oshai.kotlinlogging.KotlinLogging -import io.grpc.StatusRuntimeException -import org.vitrivr.cottontail.client.language.ddl.CreateEntity -import org.vitrivr.cottontail.core.database.Name -import org.vitrivr.cottontail.core.types.Types -import org.vitrivr.engine.core.model.descriptor.scalar.ScalarDescriptor -import org.vitrivr.engine.core.model.metamodel.Schema -import org.vitrivr.engine.plugin.cottontaildb.* -import org.vitrivr.engine.plugin.cottontaildb.descriptors.AbstractDescriptorInitializer - -private val logger: KLogger = KotlinLogging.logger {} - -/** - * A [AbstractDescriptorInitializer] implementation for [ScalarDescriptor]s. - * - * @author Ralph Gasser - * @version 1.0.0 - */ -class ScalarDescriptorInitializer>(field: Schema.Field<*, T>, connection: CottontailConnection) : AbstractDescriptorInitializer(field, connection) { - /** - * Initializes the Cottontail DB entity backing this [AbstractDescriptorInitializer]. - */ - override fun initialize() { - /* Determine type based on prototype for Schema.Field. */ - val type = this.field.analyser.prototype(this.field).toType() - - /* Prepare query. */ - val create = CreateEntity(this.entityName) - .column(Name.ColumnName.create(DESCRIPTOR_ID_COLUMN_NAME), Types.Uuid, nullable = false, primaryKey = true, autoIncrement = false) - .column(Name.ColumnName.create(RETRIEVABLE_ID_COLUMN_NAME), Types.Uuid, nullable = false, primaryKey = false, autoIncrement = false) - .column(Name.ColumnName.create(DESCRIPTOR_COLUMN_NAME), type, nullable = false, primaryKey = false, autoIncrement = false) - - try { - /* Try to create entity. */ - this.connection.client.create(create) - } catch (e: StatusRuntimeException) { - logger.error(e) { "Failed to initialize entity ${this.entityName} due to exception." } - } - } -} \ No newline at end of file diff --git a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/scalar/ScalarDescriptorProvider.kt b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/scalar/ScalarDescriptorProvider.kt index ce6b262e..b27c560c 100644 --- a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/scalar/ScalarDescriptorProvider.kt +++ b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/scalar/ScalarDescriptorProvider.kt @@ -5,15 +5,17 @@ import org.vitrivr.engine.core.database.descriptor.DescriptorProvider import org.vitrivr.engine.core.model.descriptor.scalar.ScalarDescriptor import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.plugin.cottontaildb.CottontailConnection +import org.vitrivr.engine.plugin.cottontaildb.descriptors.CottontailDescriptorInitializer +import org.vitrivr.engine.plugin.cottontaildb.descriptors.CottontailDescriptorWriter /** * A [DescriptorProvider] for [ScalarDescriptor]s. * * @author Ralph Gasser - * @version 1.0.0 + * @version 1.1.0 */ internal object ScalarDescriptorProvider : DescriptorProvider> { - override fun newInitializer(connection: Connection, field: Schema.Field<*, ScalarDescriptor<*>>) = ScalarDescriptorInitializer(field, connection as CottontailConnection) + override fun newInitializer(connection: Connection, field: Schema.Field<*, ScalarDescriptor<*>>) = CottontailDescriptorInitializer(field, connection as CottontailConnection) override fun newReader(connection: Connection, field: Schema.Field<*, ScalarDescriptor<*>>) = ScalarDescriptorReader(field, connection as CottontailConnection) - override fun newWriter(connection: Connection, field: Schema.Field<*, ScalarDescriptor<*>>) = ScalarDescriptorWriter(field, connection as CottontailConnection) + override fun newWriter(connection: Connection, field: Schema.Field<*, ScalarDescriptor<*>>) = CottontailDescriptorWriter(field, connection as CottontailConnection) } \ No newline at end of file diff --git a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/scalar/ScalarDescriptorReader.kt b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/scalar/ScalarDescriptorReader.kt index 70e952e2..46db246b 100644 --- a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/scalar/ScalarDescriptorReader.kt +++ b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/scalar/ScalarDescriptorReader.kt @@ -5,9 +5,12 @@ import org.vitrivr.cottontail.client.language.basics.expression.Literal import org.vitrivr.cottontail.client.language.basics.predicate.Compare import org.vitrivr.cottontail.core.tuple.Tuple import org.vitrivr.engine.core.model.descriptor.scalar.* +import org.vitrivr.engine.core.model.descriptor.scalar.ScalarDescriptor.Companion.VALUE_ATTRIBUTE_NAME +import org.vitrivr.engine.core.model.descriptor.vector.VectorDescriptor.Companion.VECTOR_ATTRIBUTE_NAME import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.query.Query import org.vitrivr.engine.core.model.query.bool.SimpleBooleanQuery +import org.vitrivr.engine.core.model.query.fulltext.SimpleFulltextQuery import org.vitrivr.engine.core.model.retrievable.Retrieved import org.vitrivr.engine.core.model.types.toValue import org.vitrivr.engine.plugin.cottontaildb.* @@ -29,15 +32,45 @@ class ScalarDescriptorReader(field: Schema.Field<*, ScalarDescriptor<*>>, connec * * @param query The [Query] to execute. */ - override fun query(query: Query): Sequence> { - require(query is SimpleBooleanQuery<*>) { "Query of type ${query::class} is not supported by ScalarDescriptorReader." } + override fun query(query: Query): Sequence> = when (query) { + is SimpleFulltextQuery -> this.queryFulltext(query) + is SimpleBooleanQuery<*> -> this.queryBoolean(query) + else -> throw UnsupportedOperationException("The provided query type ${query::class.simpleName} is not supported by this reader.") + } - /* Prepare query. */ + /** + * Converts the provided [Tuple] to a [ScalarDescriptor]. + * + * @param tuple The [Tuple] to convert. + * @return The resulting [ScalarDescriptor]. + */ + override fun tupleToDescriptor(tuple: Tuple): ScalarDescriptor<*> { + val retrievableId = tuple.asUuidValue(RETRIEVABLE_ID_COLUMN_NAME)?.value ?: throw IllegalArgumentException("The provided tuple is missing the required field '${RETRIEVABLE_ID_COLUMN_NAME}'.") + val descriptorId = tuple.asUuidValue(DESCRIPTOR_ID_COLUMN_NAME)?.value ?: throw IllegalArgumentException("The provided tuple is missing the required field '${DESCRIPTOR_ID_COLUMN_NAME}'.") + return when (this.prototype) { + is BooleanDescriptor -> BooleanDescriptor(retrievableId, descriptorId, tuple.asBoolean(VALUE_ATTRIBUTE_NAME)?.toValue() ?: throw IllegalArgumentException("The provided tuple is missing the required field '$VECTOR_ATTRIBUTE_NAME'.")) + is DoubleDescriptor -> DoubleDescriptor(retrievableId, descriptorId, tuple.asDouble(VALUE_ATTRIBUTE_NAME)?.toValue() ?: throw IllegalArgumentException("The provided tuple is missing the required field '$VECTOR_ATTRIBUTE_NAME'.")) + is FloatDescriptor -> FloatDescriptor(retrievableId, descriptorId, tuple.asFloat(VALUE_ATTRIBUTE_NAME)?.toValue() ?: throw IllegalArgumentException("The provided tuple is missing the required field '$VECTOR_ATTRIBUTE_NAME'.")) + is IntDescriptor -> IntDescriptor(retrievableId, descriptorId, tuple.asInt(VALUE_ATTRIBUTE_NAME)?.toValue() ?: throw IllegalArgumentException("The provided tuple is missing the required field '$VECTOR_ATTRIBUTE_NAME'.")) + is LongDescriptor -> LongDescriptor(retrievableId, descriptorId, tuple.asLong(VALUE_ATTRIBUTE_NAME)?.toValue() ?: throw IllegalArgumentException("The provided tuple is missing the required field '$VECTOR_ATTRIBUTE_NAME'.")) + is StringDescriptor -> StringDescriptor(retrievableId, descriptorId, tuple.asString(VALUE_ATTRIBUTE_NAME)?.toValue() ?: throw IllegalArgumentException("The provided tuple is missing the required field '$VECTOR_ATTRIBUTE_NAME'.")) + } + } + + /** + * Executes a [SimpleFulltextQuery] and returns a [Sequence] of [ScalarDescriptor]s. + * + * @param query The [SimpleFulltextQuery] to execute. + * @return [Sequence] of [ScalarDescriptor]s. + */ + private fun queryFulltext(query: SimpleFulltextQuery): Sequence> { val cottontailQuery = org.vitrivr.cottontail.client.language.dql.Query(this.entityName) - .select(RETRIEVABLE_ID_COLUMN_NAME) - .select(DESCRIPTOR_ID_COLUMN_NAME) - .select(DESCRIPTOR_COLUMN_NAME) - .where(Compare(Column(this.entityName.column(DESCRIPTOR_COLUMN_NAME)), query.operator(), Literal(query.value.toCottontailValue()))) + .select("*") + .fulltext(VALUE_ATTRIBUTE_NAME, query.value.value, "score") + + if (query.limit < Long.MAX_VALUE) { + cottontailQuery.limit(query.limit) + } /* Execute query. */ return this.connection.client.query(cottontailQuery).asSequence().map { @@ -46,30 +79,22 @@ class ScalarDescriptorReader(field: Schema.Field<*, ScalarDescriptor<*>>, connec } /** - * Executes the provided [Query] and returns a [Sequence] of [Retrieved]s that match it. + * Executes a [SimpleBooleanQuery] and returns a [Sequence] of [ScalarDescriptor]s. * - * @param query The [Query] to execute. + * @param query The [SimpleBooleanQuery] to execute. + * @return [Sequence] of [ScalarDescriptor]s. */ - override fun queryAndJoin(query: Query): Sequence { - TODO("Not yet implemented") - } + private fun queryBoolean(query: SimpleBooleanQuery<*>): Sequence> { + /* Prepare query. */ + val cottontailQuery = org.vitrivr.cottontail.client.language.dql.Query(this.entityName) + .select(RETRIEVABLE_ID_COLUMN_NAME) + .select(DESCRIPTOR_ID_COLUMN_NAME) + .select(VALUE_ATTRIBUTE_NAME) + .where(Compare(Column(this.entityName.column(VALUE_ATTRIBUTE_NAME)), query.operator(), Literal(query.value.toCottontailValue()))) - /** - * Converts the provided [Tuple] to a [ScalarDescriptor]. - * - * @param tuple The [Tuple] to convert. - * @return The resulting [ScalarDescriptor]. - */ - override fun tupleToDescriptor(tuple: Tuple): ScalarDescriptor<*> { - val retrievableId = tuple.asUuidValue(RETRIEVABLE_ID_COLUMN_NAME)?.value ?: throw IllegalArgumentException("The provided tuple is missing the required field '${RETRIEVABLE_ID_COLUMN_NAME}'.") - val descriptorId = tuple.asUuidValue(DESCRIPTOR_ID_COLUMN_NAME)?.value ?: throw IllegalArgumentException("The provided tuple is missing the required field '${DESCRIPTOR_ID_COLUMN_NAME}'.") - return when (this.prototype) { - is BooleanDescriptor -> BooleanDescriptor(retrievableId, descriptorId, tuple.asBoolean(DESCRIPTOR_COLUMN_NAME)?.toValue() ?: throw IllegalArgumentException("The provided tuple is missing the required field '$DESCRIPTOR_COLUMN_NAME'.")) - is DoubleDescriptor -> DoubleDescriptor(retrievableId, descriptorId, tuple.asDouble(DESCRIPTOR_COLUMN_NAME)?.toValue() ?: throw IllegalArgumentException("The provided tuple is missing the required field '$DESCRIPTOR_COLUMN_NAME'.")) - is FloatDescriptor -> FloatDescriptor(retrievableId, descriptorId, tuple.asFloat(DESCRIPTOR_COLUMN_NAME)?.toValue() ?: throw IllegalArgumentException("The provided tuple is missing the required field '$DESCRIPTOR_COLUMN_NAME'.")) - is IntDescriptor -> IntDescriptor(retrievableId, descriptorId, tuple.asInt(DESCRIPTOR_COLUMN_NAME)?.toValue() ?: throw IllegalArgumentException("The provided tuple is missing the required field '$DESCRIPTOR_COLUMN_NAME'.")) - is LongDescriptor -> LongDescriptor(retrievableId, descriptorId, tuple.asLong(DESCRIPTOR_COLUMN_NAME)?.toValue() ?: throw IllegalArgumentException("The provided tuple is missing the required field '$DESCRIPTOR_COLUMN_NAME'.")) - is StringDescriptor -> StringDescriptor(retrievableId, descriptorId, tuple.asString(DESCRIPTOR_COLUMN_NAME)?.toValue() ?: throw IllegalArgumentException("The provided tuple is missing the required field '$DESCRIPTOR_COLUMN_NAME'.")) + /* Execute query. */ + return this.connection.client.query(cottontailQuery).asSequence().map { + this.tupleToDescriptor(it) } } } diff --git a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/scalar/ScalarDescriptorWriter.kt b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/scalar/ScalarDescriptorWriter.kt deleted file mode 100644 index 261d6c83..00000000 --- a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/scalar/ScalarDescriptorWriter.kt +++ /dev/null @@ -1,100 +0,0 @@ -package org.vitrivr.engine.plugin.cottontaildb.descriptors.scalar - -import io.github.oshai.kotlinlogging.KLogger -import io.github.oshai.kotlinlogging.KotlinLogging -import io.grpc.StatusRuntimeException -import org.vitrivr.cottontail.client.language.basics.expression.Column -import org.vitrivr.cottontail.client.language.basics.expression.Literal -import org.vitrivr.cottontail.client.language.basics.predicate.Compare -import org.vitrivr.cottontail.client.language.dml.BatchInsert -import org.vitrivr.cottontail.client.language.dml.Insert -import org.vitrivr.cottontail.client.language.dml.Update -import org.vitrivr.cottontail.core.values.UuidValue -import org.vitrivr.engine.core.model.descriptor.scalar.ScalarDescriptor -import org.vitrivr.engine.core.model.metamodel.Schema -import org.vitrivr.engine.plugin.cottontaildb.* -import org.vitrivr.engine.plugin.cottontaildb.descriptors.AbstractDescriptorWriter - -private val logger: KLogger = KotlinLogging.logger {} - -/** - * An [AbstractDescriptorWriter] for [ScalarDescriptor]s. - * - * @author Ralph Gasser - * @version 1.0.0 - */ -class ScalarDescriptorWriter(field: Schema.Field<*, ScalarDescriptor<*>>, connection: CottontailConnection) : AbstractDescriptorWriter>(field, connection) { - - /** - * Adds (writes) a single [ScalarDescriptor] using this [ScalarDescriptorWriter]. - * - * @param item The [ScalarDescriptor] to write. - * @return True on success, false otherwise. - */ - override fun add(item: ScalarDescriptor<*>): Boolean { - val insert = Insert(this.entityName).values( - DESCRIPTOR_ID_COLUMN_NAME to UuidValue(item.id), - RETRIEVABLE_ID_COLUMN_NAME to UuidValue(item.retrievableId ?: throw IllegalArgumentException("A scalar descriptor must be associated with a retrievable ID.")), - DESCRIPTOR_COLUMN_NAME to item.toCottontailValue() - ) - return try { - this.connection.client.insert(insert).use { - it.hasNext() - } - } catch (e: StatusRuntimeException) { - logger.error(e) { "Failed to persist descriptor ${item.id} due to exception." } - false - } - } - - /** - * Adds (writes) a batch of [ScalarDescriptor] using this [ScalarDescriptorWriter]. - * - * @param items A [Iterable] of [ScalarDescriptor]s to write. - * @return True on success, false otherwise. - */ - override fun addAll(items: Iterable>): Boolean { - /* Prepare insert query. */ - var size = 0 - val insert = BatchInsert(this.entityName).columns(DESCRIPTOR_ID_COLUMN_NAME, RETRIEVABLE_ID_COLUMN_NAME, DESCRIPTOR_COLUMN_NAME) - for (item in items) { - size += 1 - insert.values(UuidValue(item.id.toString()), UuidValue(item.retrievableId ?: throw IllegalArgumentException("A scalar descriptor must be associated with a retrievable ID.")), item.toCottontailValue()) - } - - /* Insert values. */ - return try { - this.connection.client.insert(insert).use { - it.hasNext() - } - } catch (e: StatusRuntimeException) { - logger.error(e) { "Failed to persist $size scalar descriptors due to exception." } - false - } - } - - /** - * Updates a specific [ScalarDescriptor] using this [ScalarDescriptorWriter]. - * - * @param item A [ScalarDescriptor]s to update. - * @return True on success, false otherwise. - */ - override fun update(item: ScalarDescriptor<*>): Boolean { - val update = Update(this.entityName).where( - Compare( - Column(this.entityName.column(DESCRIPTOR_ID_COLUMN_NAME)), - Compare.Operator.EQUAL, - Literal(UuidValue(item.id)) - ) - ).values(DESCRIPTOR_COLUMN_NAME to item.toCottontailValue()) - - /* Delete values. */ - return try { - this.connection.client.update(update) - true - } catch (e: StatusRuntimeException) { - logger.error(e) { "Failed to update descriptor due to exception." } - false - } - } -} \ No newline at end of file diff --git a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/string/StringDescriptorInitializer.kt b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/string/StringDescriptorInitializer.kt deleted file mode 100644 index e3005032..00000000 --- a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/string/StringDescriptorInitializer.kt +++ /dev/null @@ -1,52 +0,0 @@ -package org.vitrivr.engine.plugin.cottontaildb.descriptors.string - -import io.github.oshai.kotlinlogging.KLogger -import io.github.oshai.kotlinlogging.KotlinLogging -import io.grpc.StatusRuntimeException -import org.vitrivr.cottontail.client.language.ddl.CreateEntity -import org.vitrivr.cottontail.client.language.ddl.CreateIndex -import org.vitrivr.cottontail.core.database.Name -import org.vitrivr.cottontail.core.types.Types -import org.vitrivr.cottontail.grpc.CottontailGrpc -import org.vitrivr.engine.core.model.descriptor.scalar.StringDescriptor -import org.vitrivr.engine.core.model.metamodel.Schema -import org.vitrivr.engine.plugin.cottontaildb.CottontailConnection -import org.vitrivr.engine.plugin.cottontaildb.DESCRIPTOR_COLUMN_NAME -import org.vitrivr.engine.plugin.cottontaildb.DESCRIPTOR_ID_COLUMN_NAME -import org.vitrivr.engine.plugin.cottontaildb.RETRIEVABLE_ID_COLUMN_NAME -import org.vitrivr.engine.plugin.cottontaildb.descriptors.AbstractDescriptorInitializer - -private val logger: KLogger = KotlinLogging.logger {} - -/** - * A [AbstractDescriptorInitializer] implementation for [StringDescriptor]s. - * - * @author Ralph Gasser - * @version 1.0.0 - */ -class StringDescriptorInitializer(field: Schema.Field<*, StringDescriptor>, connection: CottontailConnection) : AbstractDescriptorInitializer(field, connection) { - /** - * Initializes the Cottontail DB entity backing this [AbstractDescriptorInitializer]. - */ - override fun initialize() { - /* Prepare to create entity. */ - val create = CreateEntity(this.entityName) - .column(Name.ColumnName.create(DESCRIPTOR_ID_COLUMN_NAME), Types.Uuid, nullable = false, primaryKey = true, autoIncrement = false) - .column(Name.ColumnName.create(RETRIEVABLE_ID_COLUMN_NAME), Types.Uuid, nullable = false, primaryKey = false, autoIncrement = false) - .column(Name.ColumnName.create(DESCRIPTOR_COLUMN_NAME), Types.String, nullable = false, primaryKey = false, autoIncrement = false) - - try { - /* Try to create entity. */ - this.connection.client.create(create) - - /* Create entity if necessary. */ - if (this.field.parameters.containsKey("index")) { - val createIndex = CreateIndex(this.entityName, CottontailGrpc.IndexType.LUCENE) - .column(this.entityName.column(DESCRIPTOR_COLUMN_NAME)) - this.connection.client.create(createIndex) - } - } catch (e: StatusRuntimeException) { - logger.error(e) { "Failed to initialize entity ${this.entityName} due to exception." } - } - } -} \ No newline at end of file diff --git a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/string/StringDescriptorProvider.kt b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/string/StringDescriptorProvider.kt deleted file mode 100644 index 13f347eb..00000000 --- a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/string/StringDescriptorProvider.kt +++ /dev/null @@ -1,19 +0,0 @@ -package org.vitrivr.engine.plugin.cottontaildb.descriptors.string - -import org.vitrivr.engine.core.database.Connection -import org.vitrivr.engine.core.database.descriptor.DescriptorProvider -import org.vitrivr.engine.core.model.descriptor.scalar.StringDescriptor -import org.vitrivr.engine.core.model.metamodel.Schema -import org.vitrivr.engine.plugin.cottontaildb.CottontailConnection - -/** - * A [DescriptorProvider] for [StringDescriptor]s. - * - * @author Ralph Gasser - * @version 1.0.0 - */ -object StringDescriptorProvider : DescriptorProvider { - override fun newInitializer(connection: Connection, field: Schema.Field<*, StringDescriptor>) = StringDescriptorInitializer(field, connection as CottontailConnection) - override fun newReader(connection: Connection, field: Schema.Field<*, StringDescriptor>) = StringDescriptorReader(field, connection as CottontailConnection) - override fun newWriter(connection: Connection, field: Schema.Field<*, StringDescriptor>) = StringDescriptorWriter(field, connection as CottontailConnection) -} \ No newline at end of file diff --git a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/string/StringDescriptorReader.kt b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/string/StringDescriptorReader.kt deleted file mode 100644 index 365c19ec..00000000 --- a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/string/StringDescriptorReader.kt +++ /dev/null @@ -1,118 +0,0 @@ -package org.vitrivr.engine.plugin.cottontaildb.descriptors.string - -import org.vitrivr.cottontail.client.language.basics.expression.Column -import org.vitrivr.cottontail.client.language.basics.expression.Literal -import org.vitrivr.cottontail.client.language.basics.predicate.Compare -import org.vitrivr.cottontail.core.tuple.Tuple -import org.vitrivr.engine.core.model.descriptor.scalar.StringDescriptor -import org.vitrivr.engine.core.model.metamodel.Schema -import org.vitrivr.engine.core.model.query.Query -import org.vitrivr.engine.core.model.query.bool.SimpleBooleanQuery -import org.vitrivr.engine.core.model.query.fulltext.SimpleFulltextQuery -import org.vitrivr.engine.core.model.retrievable.Retrieved -import org.vitrivr.engine.core.model.retrievable.attributes.ScoreAttribute -import org.vitrivr.engine.core.model.types.Value -import org.vitrivr.engine.core.model.types.toValue -import org.vitrivr.engine.plugin.cottontaildb.* -import org.vitrivr.engine.plugin.cottontaildb.descriptors.AbstractDescriptorReader - - -/** - * An [AbstractDescriptorReader] for [StringDescriptor]s. - * - * @author Ralph Gasser - * @version 2.0.0 - */ -class StringDescriptorReader(field: Schema.Field<*, StringDescriptor>, connection: CottontailConnection) : AbstractDescriptorReader(field, connection) { - /** - * Executes the provided [Query] and returns a [Sequence] of [Retrieved]s that match it. - * - * @param query The [Query] to execute. - */ - override fun query(query: Query): Sequence { - /* Prepare query. */ - val cottontailQuery = org.vitrivr.cottontail.client.language.dql.Query(this.entityName).select(RETRIEVABLE_ID_COLUMN_NAME) - when (query) { - is SimpleFulltextQuery -> { - cottontailQuery - .select("*") - .fulltext(DESCRIPTOR_COLUMN_NAME, query.value.value, "score") - if (query.limit < Long.MAX_VALUE) { - cottontailQuery.limit(query.limit) - } - } - - is SimpleBooleanQuery<*> -> { - require(query.value is Value.String) { "StringDescriptorReader can only perform comparisons to string values." } - cottontailQuery.where(Compare(Column(this.entityName.column(DESCRIPTOR_COLUMN_NAME)), query.operator(), Literal(query.value.toCottontailValue()))) - } - else -> throw IllegalArgumentException("Query of typ ${query::class} is not supported by StringDescriptorReader.") - } - - /* Execute query. */ - return this.connection.client.query(cottontailQuery).asSequence().map { tuple -> - this.tupleToDescriptor(tuple) - } - } - - /** - * Returns a [Sequence] of all [Retrieved]s that match the given [Query]. - * - * Implicitly, this methods executes a [query] and then JOINS the result with the [Retrieved]s. - * - * @param query The [Query] that should be executed. - * @return [Sequence] of [Retrieved]. - */ - override fun queryAndJoin(query: Query): Sequence { - /* Prepare query. */ - val cottontailQuery = org.vitrivr.cottontail.client.language.dql.Query(this.entityName).select(RETRIEVABLE_ID_COLUMN_NAME) - when (query) { - is SimpleFulltextQuery -> { - cottontailQuery - .select(DESCRIPTOR_COLUMN_NAME) - .select(DESCRIPTOR_ID_COLUMN_NAME) - .fulltext(DESCRIPTOR_COLUMN_NAME, query.value.value, "score") - - if (query.limit < Long.MAX_VALUE) { - cottontailQuery.limit(query.limit) - } - } - - is SimpleBooleanQuery<*> -> { - require(query.value is Value.String) { "StringDescriptorReader can only perform comparisons to string values." } - cottontailQuery.where(Compare(Column(this.entityName.column(DESCRIPTOR_COLUMN_NAME)), query.operator(), Literal(query.value.toCottontailValue()))) - } - - else -> throw IllegalArgumentException("Query of typ ${query::class} is not supported by StringDescriptorReader.") - } - - /* Execute query. */ - val descriptors = this.connection.client.query(cottontailQuery).asSequence().map { tuple -> - val scoreIndex = tuple.indexOf(SCORE_COLUMN_NAME) - tupleToDescriptor(tuple) to if (scoreIndex > -1) { - tuple.asDouble(SCORE_COLUMN_NAME)?.let { ScoreAttribute.Unbound(it.toFloat()) } - } else { - null - } - }.toList() - if (descriptors.isEmpty()) return emptySequence() - - /* Fetch retrievable ids. */ - val retrievables = this.fetchRetrievable(descriptors.mapNotNull { it.first.retrievableId }.toSet()) - return descriptors.asSequence().mapNotNull { descriptor -> - val retrievable = retrievables[descriptor.first.retrievableId] ?: return@mapNotNull null - - /* Append descriptor and score attribute. */ - retrievable.addDescriptor(descriptor.first) - descriptor.second?.let { retrievable.addAttribute(it) } - retrievable - } - } - - override fun tupleToDescriptor(tuple: Tuple): StringDescriptor { - val retrievableId = tuple.asUuidValue(RETRIEVABLE_ID_COLUMN_NAME)?.value ?: throw IllegalArgumentException("The provided tuple is missing the required field '${RETRIEVABLE_ID_COLUMN_NAME}'.") - val descriptorId = tuple.asUuidValue(DESCRIPTOR_ID_COLUMN_NAME)?.value ?: throw IllegalArgumentException("The provided tuple is missing the required field '${DESCRIPTOR_ID_COLUMN_NAME}'.") - val value = tuple.asString(DESCRIPTOR_COLUMN_NAME)?.toValue() ?: throw IllegalArgumentException("The provided tuple is missing the required field '$DESCRIPTOR_COLUMN_NAME'.") - return StringDescriptor(descriptorId, retrievableId, value) - } -} diff --git a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/string/StringDescriptorWriter.kt b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/string/StringDescriptorWriter.kt deleted file mode 100644 index ad3d7c3f..00000000 --- a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/string/StringDescriptorWriter.kt +++ /dev/null @@ -1,101 +0,0 @@ -package org.vitrivr.engine.plugin.cottontaildb.descriptors.string - -import io.github.oshai.kotlinlogging.KLogger -import io.github.oshai.kotlinlogging.KotlinLogging -import io.grpc.StatusRuntimeException -import org.vitrivr.cottontail.client.language.basics.expression.Column -import org.vitrivr.cottontail.client.language.basics.expression.Literal -import org.vitrivr.cottontail.client.language.basics.predicate.Compare -import org.vitrivr.cottontail.client.language.dml.BatchInsert -import org.vitrivr.cottontail.client.language.dml.Insert -import org.vitrivr.cottontail.client.language.dml.Update -import org.vitrivr.cottontail.core.values.UuidValue -import org.vitrivr.engine.core.model.descriptor.scalar.StringDescriptor -import org.vitrivr.engine.core.model.metamodel.Schema -import org.vitrivr.engine.plugin.cottontaildb.* -import org.vitrivr.engine.plugin.cottontaildb.descriptors.AbstractDescriptorWriter -import org.vitrivr.engine.plugin.cottontaildb.descriptors.vector.VectorDescriptorWriter - -private val logger: KLogger = KotlinLogging.logger {} - -/** - * An [AbstractDescriptorWriter] for [StringDescriptor]s. - * - * @author Ralph Gasser - * @version 1.0.0 - */ -class StringDescriptorWriter(field: Schema.Field<*, StringDescriptor>, connection: CottontailConnection) : AbstractDescriptorWriter(field, connection) { - - /** - * Adds (writes) a single [StringDescriptor] using this [StringDescriptorWriter]. - * - * @param item The [StringDescriptor] to write. - * @return True on success, false otherwise. - */ - override fun add(item: StringDescriptor): Boolean { - val insert = Insert(this.entityName).values( - DESCRIPTOR_ID_COLUMN_NAME to UuidValue(item.id), - RETRIEVABLE_ID_COLUMN_NAME to UuidValue(item.retrievableId ?: throw IllegalArgumentException("A string descriptor must be associated with a retrievable ID.")), - DESCRIPTOR_COLUMN_NAME to item.toCottontailValue() - ) - return try { - this.connection.client.insert(insert).use { - it.hasNext() - } - } catch (e: StatusRuntimeException) { - logger.error(e) { "Failed to persist descriptor ${item.id} due to exception." } - false - } - } - - /** - * Adds (writes) a batch of [StringDescriptor] using this [StringDescriptorWriter]. - * - * @param items A [Iterable] of [StringDescriptor]s to write. - * @return True on success, false otherwise. - */ - override fun addAll(items: Iterable): Boolean { - /* Prepare insert query. */ - var size = 0 - val insert = BatchInsert(this.entityName).columns(DESCRIPTOR_ID_COLUMN_NAME, RETRIEVABLE_ID_COLUMN_NAME, DESCRIPTOR_COLUMN_NAME) - for (item in items) { - size += 1 - insert.values(UuidValue(item.id.toString()), UuidValue(item.retrievableId ?: throw IllegalArgumentException("A string descriptor must be associated with a retrievable ID.")), item.toCottontailValue()) - } - - /* Insert values. */ - return try { - this.connection.client.insert(insert).use { - it.hasNext() - } - } catch (e: StatusRuntimeException) { - logger.error(e) { "Failed to persist $size string descriptors due to exception." } - false - } - } - - /** - * Updates a specific [StringDescriptor] using this [VectorDescriptorWriter]. - * - * @param item A [StringDescriptor]s to update. - * @return True on success, false otherwise. - */ - override fun update(item: StringDescriptor): Boolean { - val update = Update(this.entityName).where( - Compare( - Column(this.entityName.column(DESCRIPTOR_ID_COLUMN_NAME)), - Compare.Operator.EQUAL, - Literal(UuidValue(item.id)) - ) - ).values(DESCRIPTOR_COLUMN_NAME to item.toCottontailValue()) - - /* Updates values. */ - return try { - this.connection.client.update(update) - true - } catch (e: StatusRuntimeException) { - logger.error(e) { "Failed to update descriptor due to exception." } - false - } - } -} \ No newline at end of file diff --git a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/struct/StructDescriptorInitializer.kt b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/struct/StructDescriptorInitializer.kt deleted file mode 100644 index 4a46065c..00000000 --- a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/struct/StructDescriptorInitializer.kt +++ /dev/null @@ -1,85 +0,0 @@ -package org.vitrivr.engine.plugin.cottontaildb.descriptors.struct - -import io.github.oshai.kotlinlogging.KLogger -import io.github.oshai.kotlinlogging.KotlinLogging -import io.grpc.StatusRuntimeException -import org.vitrivr.cottontail.client.language.ddl.CreateEntity -import org.vitrivr.cottontail.core.database.Name -import org.vitrivr.cottontail.core.types.Types -import org.vitrivr.engine.core.model.descriptor.struct.StructDescriptor -import org.vitrivr.engine.core.model.metamodel.Schema -import org.vitrivr.engine.core.model.types.Type -import org.vitrivr.engine.plugin.cottontaildb.CottontailConnection -import org.vitrivr.engine.plugin.cottontaildb.DESCRIPTOR_ID_COLUMN_NAME -import org.vitrivr.engine.plugin.cottontaildb.RETRIEVABLE_ID_COLUMN_NAME -import org.vitrivr.engine.plugin.cottontaildb.descriptors.AbstractDescriptorInitializer - -private val logger: KLogger = KotlinLogging.logger {} - -/** - * A [AbstractDescriptorInitializer] implementation for [StructDescriptor]s. - * - * @author Ralph Gasser - * @version 1.0.0 - */ -class StructDescriptorInitializer(field: Schema.Field<*, StructDescriptor>, connection: CottontailConnection) : AbstractDescriptorInitializer(field, connection) { - /** - * Initializes the Cottontail DB entity backing this [AbstractDescriptorInitializer]. - */ - override fun initialize() { - /* Prepare query. */ - val create = CreateEntity(this.entityName) - .column(Name.ColumnName.create(DESCRIPTOR_ID_COLUMN_NAME), Types.Uuid, nullable = false, primaryKey = true, autoIncrement = false) - .column(Name.ColumnName.create(RETRIEVABLE_ID_COLUMN_NAME), Types.Uuid, nullable = false, primaryKey = false, autoIncrement = false) - - - for (field in this.field.analyser.prototype(this.field).schema()) { - require(field.dimensions.size <= 1) { "Cottontail DB currently doesn't support tensor types."} - val vector = field.dimensions.size == 1 - val type = when (field.type) { - Type.STRING -> Types.String - Type.BYTE -> Types.Byte - Type.SHORT -> Types.Short - Type.BOOLEAN -> if (vector) { - Types.BooleanVector(field.dimensions[0]) - } else { - Types.Boolean - } - - Type.INT -> if (vector) { - Types.IntVector(field.dimensions[0]) - } else { - Types.Int - } - - Type.LONG -> if (vector) { - Types.LongVector(field.dimensions[0]) - } else { - Types.Long - } - - Type.FLOAT -> if (vector) { - Types.FloatVector(field.dimensions[0]) - } else { - Types.Float - } - - Type.DOUBLE -> if (vector) { - Types.DoubleVector(field.dimensions[0]) - } else { - Types.Double - } - - Type.DATETIME -> Types.Date - } - create.column(Name.ColumnName.create(field.name), type, nullable = field.nullable, primaryKey = false, autoIncrement = false) - } - - try { - /* Try to create entity. */ - this.connection.client.create(create) - } catch (e: StatusRuntimeException) { - logger.error(e) { "Failed to initialize entity ${this.entityName} due to exception." } - } - } -} diff --git a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/struct/StructDescriptorProvider.kt b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/struct/StructDescriptorProvider.kt index 68022c70..7629868e 100644 --- a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/struct/StructDescriptorProvider.kt +++ b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/struct/StructDescriptorProvider.kt @@ -6,15 +6,17 @@ import org.vitrivr.engine.core.model.descriptor.struct.LabelDescriptor import org.vitrivr.engine.core.model.descriptor.struct.StructDescriptor import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.plugin.cottontaildb.CottontailConnection +import org.vitrivr.engine.plugin.cottontaildb.descriptors.CottontailDescriptorInitializer +import org.vitrivr.engine.plugin.cottontaildb.descriptors.CottontailDescriptorWriter /** * A [DescriptorProvider] for [LabelDescriptor]s. * * @author Ralph Gasser - * @version 1.0.0 + * @version 1.1.0 */ object StructDescriptorProvider : DescriptorProvider { - override fun newInitializer(connection: Connection, field: Schema.Field<*, StructDescriptor>) = StructDescriptorInitializer(field, connection as CottontailConnection) + override fun newInitializer(connection: Connection, field: Schema.Field<*, StructDescriptor>) = CottontailDescriptorInitializer(field, connection as CottontailConnection) override fun newReader(connection: Connection, field: Schema.Field<*, StructDescriptor>) = StructDescriptorReader(field, connection as CottontailConnection) - override fun newWriter(connection: Connection, field: Schema.Field<*, StructDescriptor>) = StructDescriptorWriter(field, connection as CottontailConnection) + override fun newWriter(connection: Connection, field: Schema.Field<*, StructDescriptor>) = CottontailDescriptorWriter(field, connection as CottontailConnection) } \ No newline at end of file diff --git a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/struct/StructDescriptorReader.kt b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/struct/StructDescriptorReader.kt index 2206d8b8..c0df79ec 100644 --- a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/struct/StructDescriptorReader.kt +++ b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/struct/StructDescriptorReader.kt @@ -5,9 +5,9 @@ import org.vitrivr.cottontail.client.language.basics.expression.Literal import org.vitrivr.cottontail.client.language.basics.predicate.Compare import org.vitrivr.cottontail.core.tuple.Tuple import org.vitrivr.cottontail.core.types.Types +import org.vitrivr.engine.core.model.descriptor.AttributeName import org.vitrivr.engine.core.model.descriptor.struct.LabelDescriptor import org.vitrivr.engine.core.model.descriptor.struct.StructDescriptor -import org.vitrivr.engine.core.model.descriptor.struct.metadata.source.MapStructDescriptor import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.query.Query import org.vitrivr.engine.core.model.query.bool.SimpleBooleanQuery @@ -16,7 +16,6 @@ import org.vitrivr.engine.core.model.retrievable.Retrieved import org.vitrivr.engine.core.model.types.Value import org.vitrivr.engine.plugin.cottontaildb.* import org.vitrivr.engine.plugin.cottontaildb.descriptors.AbstractDescriptorReader -import java.util.* import kotlin.reflect.full.primaryConstructor /** @@ -31,9 +30,8 @@ class StructDescriptorReader(field: Schema.Field<*, StructDescriptor>, connectio private val fieldMap = mutableListOf>>() init { val prototype = this.field.analyser.prototype(this.field) - for (f in prototype.schema()) { - require(f.dimensions.size <= 1) { "Cottontail DB currently doesn't support tensor types."} - this.fieldMap.add(f.name to f.toCottontailType()) + for (f in prototype.layout()) { + this.fieldMap.add(f.name to f.type.toCottontailType()) } } @@ -43,33 +41,10 @@ class StructDescriptorReader(field: Schema.Field<*, StructDescriptor>, connectio * @param query The [Query] to execute. * @return [Sequence] of [StructDescriptor]s that match the query. */ - override fun query(query: Query): Sequence { - /* Prepare query. */ - val cottontailQuery = org.vitrivr.cottontail.client.language.dql.Query(this.entityName).select(RETRIEVABLE_ID_COLUMN_NAME).select(DESCRIPTOR_ID_COLUMN_NAME) - for ((name, _) in this.fieldMap) { - cottontailQuery.select(name) - } - - when (query) { - is SimpleFulltextQuery -> { - require(query.attributeName != null) { "Fulltext query on a struct field requires specification of a field's attribute name." } - cottontailQuery.fulltext(query.attributeName!!, query.value.value, SCORE_COLUMN_NAME) - if (query.limit < Long.MAX_VALUE) { - cottontailQuery.limit(query.limit) - } - } - - is SimpleBooleanQuery<*> -> { - require(query.attributeName != null){"Boolean query on a struct field requires specification of a field's attribute name."} - cottontailQuery.where(Compare(Column(query.attributeName!!), query.operator(), Literal(query.value.toCottontailValue()))) - } - else -> throw IllegalArgumentException("Query of typ ${query::class} is not supported by StringDescriptorReader.") - } - - /* Execute query. */ - return this.connection.client.query(cottontailQuery).asSequence().map { tuple -> - this.tupleToDescriptor(tuple) - } + override fun query(query: Query): Sequence = when (query) { + is SimpleFulltextQuery -> this.queryFulltext(query) + is SimpleBooleanQuery<*> -> this.queryBoolean(query) + else -> throw UnsupportedOperationException("The provided query type ${query::class.simpleName} is not supported by this reader.") } /** @@ -79,42 +54,33 @@ class StructDescriptorReader(field: Schema.Field<*, StructDescriptor>, connectio * @return The resulting [StructDescriptor]. */ override fun tupleToDescriptor(tuple: Tuple): StructDescriptor { - return if (this.field.analyser.descriptorClass == MapStructDescriptor::class) { - this.tupleToMapStructDescriptor(tuple) - }else{ - this.tupleToStructDescriptor(tuple) - } - } - - private fun tupleToStructDescriptor(tuple: Tuple): StructDescriptor{ val constructor = this.field.analyser.descriptorClass.primaryConstructor ?: throw IllegalStateException("Provided type ${this.field.analyser.descriptorClass} does not have a primary constructor.") + val valueMap = mutableMapOf>() val parameters: MutableList = mutableListOf( tuple.asUuidValue(DESCRIPTOR_ID_COLUMN_NAME)?.value ?: throw IllegalArgumentException("The provided tuple is missing the required field '${DESCRIPTOR_ID_COLUMN_NAME}'."), tuple.asUuidValue(RETRIEVABLE_ID_COLUMN_NAME)?.value ?: throw IllegalArgumentException("The provided tuple is missing the required field '${RETRIEVABLE_ID_COLUMN_NAME}'."), + valueMap ) /* Append dynamic parameters of struct. */ for ((name, type) in this.fieldMap) { - parameters.add( - when(type) { - Types.Boolean -> tuple.asBoolean(name)?.let { Value.Boolean(it) } - Types.Date -> tuple.asDate(name) - Types.Byte -> tuple.asByte(name)?.let { Value.Byte(it) } - Types.Double -> tuple.asDouble(name)?.let { Value.Double(it) } - Types.Float -> tuple.asFloat(name)?.let { Value.Float(it) } - Types.Int -> tuple.asInt(name)?.let { Value.Int(it) } - Types.Long -> tuple.asLong(name)?.let { Value.Long(it) } - Types.Short -> tuple.asShort(name)?.let { Value.Short(it) } - Types.String -> tuple.asString(name)?.let { Value.String(it) } - Types.Uuid -> UUID.fromString(tuple.asString(name)) - is Types.BooleanVector -> tuple.asBooleanVector(name)?.let { List(it.size) { i -> it[i] } } - is Types.DoubleVector -> tuple.asBooleanVector(name)?.let { List(it.size) { i -> it[i] } } - is Types.FloatVector -> tuple.asBooleanVector(name)?.let { List(it.size) { i -> it[i] } } - is Types.IntVector -> tuple.asIntVector(name)?.let { List(it.size) { i -> it[i] } } - is Types.LongVector -> tuple.asLongVector(name)?.let { List(it.size) { i -> it[i] } } - else -> throw IllegalArgumentException("Type $type is not supported by StructDescriptorReader.") - } - ) + valueMap[name] = when (type) { + Types.Boolean -> tuple.asBoolean(name)?.let { Value.Boolean(it) } + Types.Date -> tuple.asDate(name)?.let { Value.DateTime(it) } + Types.Byte -> tuple.asByte(name)?.let { Value.Byte(it) } + Types.Double -> tuple.asDouble(name)?.let { Value.Double(it) } + Types.Float -> tuple.asFloat(name)?.let { Value.Float(it) } + Types.Int -> tuple.asInt(name)?.let { Value.Int(it) } + Types.Long -> tuple.asLong(name)?.let { Value.Long(it) } + Types.Short -> tuple.asShort(name)?.let { Value.Short(it) } + Types.String -> tuple.asString(name)?.let { Value.String(it) } + is Types.BooleanVector -> tuple.asBooleanVector(name)?.let { Value.BooleanVector(it) } + is Types.DoubleVector -> tuple.asDoubleVector(name)?.let { Value.DoubleVector(it) } + is Types.FloatVector -> tuple.asFloatVector(name)?.let { Value.FloatVector(it) } + is Types.IntVector -> tuple.asIntVector(name)?.let { Value.IntVector(it) } + is Types.LongVector -> tuple.asLongVector(name)?.let { Value.LongVector(it) } + else -> throw IllegalArgumentException("Type $type is not supported by StructDescriptorReader.") + } as Value<*> } parameters.add(field) //add field information, as this is for all StructDescriptors the last constructor argument. @@ -123,57 +89,46 @@ class StructDescriptorReader(field: Schema.Field<*, StructDescriptor>, connectio return constructor.call(*parameters.toTypedArray()) } - private fun tupleToMapStructDescriptor(tuple: Tuple): MapStructDescriptor{ - require(this.field.analyser.descriptorClass == MapStructDescriptor::class){"Must be MapStructDescriptor, but '${this.field.analyser.descriptorClass}' given"} - val constructor = this.field.analyser.descriptorClass.primaryConstructor ?: throw IllegalStateException("Provided type ${this.field.analyser.descriptorClass} does not have a primary constructor.") - val parameters: MutableList = mutableListOf( - tuple.asUuidValue(DESCRIPTOR_ID_COLUMN_NAME)?.value ?: throw IllegalArgumentException("The provided tuple is missing the required field '${DESCRIPTOR_ID_COLUMN_NAME}'."), - tuple.asUuidValue(RETRIEVABLE_ID_COLUMN_NAME)?.value ?: throw IllegalArgumentException("The provided tuple is missing the required field '${RETRIEVABLE_ID_COLUMN_NAME}'."), - ) - - /* Third argument: columnTypes */ - parameters.add(this.field.parameters) - - /* Fourth argument: columnValues */ - val values: MutableList = mutableListOf() - /* Append dynamic parameters of struct. */ - for ((name, type) in this.fieldMap) { - values.add( - when(type) { - Types.Boolean -> tuple.asBoolean(name)?.let { Value.Boolean(it) } - Types.Date -> tuple.asDate(name) - Types.Byte -> tuple.asByte(name)?.let { Value.Byte(it) } - Types.Double -> tuple.asDouble(name)?.let { Value.Double(it) } - Types.Float -> tuple.asFloat(name)?.let { Value.Float(it) } - Types.Int -> tuple.asInt(name)?.let { Value.Int(it) } - Types.Long -> tuple.asLong(name)?.let { Value.Long(it) } - Types.Short -> tuple.asShort(name)?.let { Value.Short(it) } - Types.String -> tuple.asString(name)?.let { Value.String(it) } - Types.Uuid -> UUID.fromString(tuple.asString(name)) - is Types.BooleanVector -> tuple.asBooleanVector(name)?.let { List(it.size) { i -> it[i] } } - is Types.DoubleVector -> tuple.asBooleanVector(name)?.let { List(it.size) { i -> it[i] } } - is Types.FloatVector -> tuple.asBooleanVector(name)?.let { List(it.size) { i -> it[i] } } - is Types.IntVector -> tuple.asIntVector(name)?.let { List(it.size) { i -> it[i] } } - is Types.LongVector -> tuple.asLongVector(name)?.let { List(it.size) { i -> it[i] } } - else -> throw IllegalArgumentException("Type $type is not supported by StructDescriptorReader.") - } - ) + /** + * Executes a [SimpleFulltextQuery] and returns a [Sequence] of [StructDescriptor]s. + * + * @param query The [SimpleFulltextQuery] to execute. + * @return [Sequence] of [StructDescriptor]s. + */ + private fun queryFulltext(query: SimpleFulltextQuery): Sequence { + require(query.attributeName != null) { "Fulltext query on a struct field requires specification of a field's attribute name." } + val cottontailQuery = org.vitrivr.cottontail.client.language.dql.Query(this.entityName).select(RETRIEVABLE_ID_COLUMN_NAME).select(DESCRIPTOR_ID_COLUMN_NAME) + for ((name, _) in this.fieldMap) { + cottontailQuery.select(name) } - require(values.size == this.field.parameters.size){"Some data was missing from the DB. Field definition and values from DB sizes differ!"} - val columnValues: MutableMap = mutableMapOf() - val columnKeys = this.field.parameters.keys.toTypedArray() - for(i in 0 until values.size){ - columnValues[columnKeys[i]] = values[i] + cottontailQuery.fulltext(query.attributeName!!, query.value.value, SCORE_COLUMN_NAME) + if (query.limit < Long.MAX_VALUE) { + cottontailQuery.limit(query.limit) } - parameters.add(columnValues) - /* Fifth argument: transient */ - parameters.add(false) //add 'transient' flag to false, since the results were actually retrieved + /* Execute query. */ + return this.connection.client.query(cottontailQuery).asSequence().map { + this.tupleToDescriptor(it) + } + } - /* Sixth argument: field */ - parameters.add(this.field) + /** + * Executes a [SimpleBooleanQuery] and returns a [Sequence] of [StructDescriptor]s. + * + * @param query The [SimpleBooleanQuery] to execute. + * @return [Sequence] of [StructDescriptor]s. + */ + private fun queryBoolean(query: SimpleBooleanQuery<*>): Sequence { + require(query.attributeName != null) { "Boolean query on a struct field requires specification of a field's attribute name." } + val cottontailQuery = org.vitrivr.cottontail.client.language.dql.Query(this.entityName).select(RETRIEVABLE_ID_COLUMN_NAME).select(DESCRIPTOR_ID_COLUMN_NAME) + for ((name, _) in this.fieldMap) { + cottontailQuery.select(name) + } + cottontailQuery.where(Compare(Column(query.attributeName!!), query.operator(), Literal(query.value.toCottontailValue()))) - /* Call constructor. */ - return constructor.call(*parameters.toTypedArray()) as MapStructDescriptor + /* Execute query. */ + return this.connection.client.query(cottontailQuery).asSequence().map { + this.tupleToDescriptor(it) + } } } diff --git a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/struct/StructDescriptorWriter.kt b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/struct/StructDescriptorWriter.kt deleted file mode 100644 index 3eff0a45..00000000 --- a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/struct/StructDescriptorWriter.kt +++ /dev/null @@ -1,133 +0,0 @@ -package org.vitrivr.engine.plugin.cottontaildb.descriptors.struct - -import io.github.oshai.kotlinlogging.KLogger -import io.github.oshai.kotlinlogging.KotlinLogging -import io.grpc.StatusRuntimeException -import org.vitrivr.cottontail.client.language.basics.expression.Column -import org.vitrivr.cottontail.client.language.basics.expression.Literal -import org.vitrivr.cottontail.client.language.basics.predicate.Compare -import org.vitrivr.cottontail.client.language.dml.BatchInsert -import org.vitrivr.cottontail.client.language.dml.Insert -import org.vitrivr.cottontail.client.language.dml.Update -import org.vitrivr.cottontail.core.values.UuidValue -import org.vitrivr.engine.core.model.descriptor.struct.StructDescriptor -import org.vitrivr.engine.core.model.metamodel.Schema -import org.vitrivr.engine.plugin.cottontaildb.CottontailConnection -import org.vitrivr.engine.plugin.cottontaildb.DESCRIPTOR_ID_COLUMN_NAME -import org.vitrivr.engine.plugin.cottontaildb.RETRIEVABLE_ID_COLUMN_NAME -import org.vitrivr.engine.plugin.cottontaildb.descriptors.AbstractDescriptorWriter -import org.vitrivr.engine.plugin.cottontaildb.toCottontailValue - -private val logger: KLogger = KotlinLogging.logger {} - -/** - * An [AbstractDescriptorWriter] for [StructDescriptor]s. - * - * @author Ralph Gasser - * @version 1.0.0 - */ -class StructDescriptorWriter(field: Schema.Field<*, StructDescriptor>, connection: CottontailConnection) : AbstractDescriptorWriter(field, connection) { - - /** - * Adds (writes) a single [StructDescriptor] using this [StructDescriptorWriter]. - * - * @param item The [StructDescriptor] to write. - * @return True on success, false otherwise. - */ - override fun add(item: StructDescriptor): Boolean { - val insert = Insert(this.entityName).values( - DESCRIPTOR_ID_COLUMN_NAME to UuidValue(item.id), - RETRIEVABLE_ID_COLUMN_NAME to UuidValue(item.retrievableId ?: throw IllegalArgumentException("A struct descriptor must be associated with a retrievable ID.")), - ) - - /* Append fields. */ - for ((field, value) in item.values()) { - insert.value(field, value?.toCottontailValue()) - } - - return try { - this.connection.client.insert(insert).use { - it.hasNext() - } - } catch (e: StatusRuntimeException) { - logger.error(e) { "Failed to persist descriptor ${item.id} due to exception." } - false - } - } - - /** - * Adds (writes) a batch of [StructDescriptor] using this [StructDescriptorWriter]. - * - * @param items A [Iterable] of [StructDescriptor]s to write. - * @return True on success, false otherwise. - */ - override fun addAll(items: Iterable): Boolean { - /* Prepare insert query. */ - var index = 0 - val insert = BatchInsert(this.entityName) - - /* Insert values. */ - for (item in items) { - val values = item.values() - if (index == 0) { - val columns = Array(values.size + 2) { - when (it) { - 0 -> DESCRIPTOR_ID_COLUMN_NAME - 1 -> RETRIEVABLE_ID_COLUMN_NAME - else -> values[it - 2].first - } - } - insert.columns(*columns) - } - val inserts: Array = Array(values.size + 2) { - when (it) { - 0 -> UuidValue(item.id) - 1 -> item.retrievableId?.let { v -> UuidValue(v) } - else -> values[it - 2].second?.toCottontailValue() - } - } - insert.any(*inserts) - index += 1 - } - - /* Insert values. */ - return try { - this.connection.client.insert(insert).use { - it.hasNext() - } - } catch (e: StatusRuntimeException) { - logger.error(e) { "Failed to persist ${index + 1} scalar descriptors due to exception." } - false - } - } - - /** - * Updates a specific [StructDescriptor] using this [StructDescriptorWriter]. - * - * @param item A [StructDescriptor]s to update. - * @return True on success, false otherwise. - */ - override fun update(item: StructDescriptor): Boolean { - val update = Update(this.entityName).where( - Compare( - Column(this.entityName.column(DESCRIPTOR_ID_COLUMN_NAME)), - Compare.Operator.EQUAL, - Literal(UuidValue(item.id)) - ) - ) - - /* Append values. */ - for ((field, value) in item.values()) { - update.any(field to value?.toCottontailValue()) - } - - /* Update values. */ - return try { - this.connection.client.update(update) - true - } catch (e: StatusRuntimeException) { - logger.error(e) { "Failed to update descriptor due to exception." } - false - } - } -} \ No newline at end of file diff --git a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/vector/VectorDescriptorInitializer.kt b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/vector/VectorDescriptorInitializer.kt deleted file mode 100644 index 8f1e1d60..00000000 --- a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/vector/VectorDescriptorInitializer.kt +++ /dev/null @@ -1,40 +0,0 @@ -package org.vitrivr.engine.plugin.cottontaildb.descriptors.vector - -import io.github.oshai.kotlinlogging.KLogger -import io.github.oshai.kotlinlogging.KotlinLogging -import io.grpc.StatusRuntimeException -import org.vitrivr.cottontail.client.language.ddl.CreateEntity -import org.vitrivr.cottontail.core.database.Name -import org.vitrivr.cottontail.core.types.Types -import org.vitrivr.engine.core.model.descriptor.vector.VectorDescriptor -import org.vitrivr.engine.core.model.metamodel.Schema -import org.vitrivr.engine.plugin.cottontaildb.* -import org.vitrivr.engine.plugin.cottontaildb.descriptors.AbstractDescriptorInitializer - -private val logger: KLogger = KotlinLogging.logger {} - - -/** - * A [AbstractDescriptorInitializer] implementation for [VectorDescriptor]s. - * - * @author Ralph Gasser - * @version 1.0.0 - */ -internal class VectorDescriptorInitializer(field: Schema.Field<*, VectorDescriptor<*>>, connection: CottontailConnection) : AbstractDescriptorInitializer>(field, connection) { - /** - * Initializes the Cottontail DB entity backing this [AbstractDescriptorInitializer]. - */ - override fun initialize() { - val type = this.field.analyser.prototype(this.field).toType() - val create = CreateEntity(this.entityName) - .column(Name.ColumnName.create(DESCRIPTOR_ID_COLUMN_NAME), Types.Uuid, nullable = false, primaryKey = true, autoIncrement = false) - .column(Name.ColumnName.create(RETRIEVABLE_ID_COLUMN_NAME), Types.Uuid, nullable = false, primaryKey = false, autoIncrement = false) - .column(Name.ColumnName.create(DESCRIPTOR_COLUMN_NAME), type, nullable = false, primaryKey = false, autoIncrement = false) - - try { - this.connection.client.create(create).close() - } catch (e: StatusRuntimeException) { - logger.error(e) { "Failed to initialize entity ${this.entityName} due to exception." } - } - } -} \ No newline at end of file diff --git a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/vector/VectorDescriptorProvider.kt b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/vector/VectorDescriptorProvider.kt index 6d2b8a55..ec87c41e 100644 --- a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/vector/VectorDescriptorProvider.kt +++ b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/vector/VectorDescriptorProvider.kt @@ -5,6 +5,8 @@ import org.vitrivr.engine.core.database.descriptor.DescriptorProvider import org.vitrivr.engine.core.model.descriptor.vector.VectorDescriptor import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.plugin.cottontaildb.CottontailConnection +import org.vitrivr.engine.plugin.cottontaildb.descriptors.CottontailDescriptorInitializer +import org.vitrivr.engine.plugin.cottontaildb.descriptors.CottontailDescriptorWriter /** * A [DescriptorProvider] for [VectorDescriptor]s. @@ -13,7 +15,7 @@ import org.vitrivr.engine.plugin.cottontaildb.CottontailConnection * @version 1.0.0 */ internal object VectorDescriptorProvider : DescriptorProvider> { - override fun newInitializer(connection: Connection, field: Schema.Field<*, VectorDescriptor<*>>) = VectorDescriptorInitializer(field, connection as CottontailConnection) + override fun newInitializer(connection: Connection, field: Schema.Field<*, VectorDescriptor<*>>) = CottontailDescriptorInitializer(field, connection as CottontailConnection) override fun newReader(connection: Connection, field: Schema.Field<*, VectorDescriptor<*>>) = VectorDescriptorReader(field, connection as CottontailConnection) - override fun newWriter(connection: Connection, field: Schema.Field<*, VectorDescriptor<*>>) = VectorDescriptorWriter(field, connection as CottontailConnection) + override fun newWriter(connection: Connection, field: Schema.Field<*, VectorDescriptor<*>>) = CottontailDescriptorWriter(field, connection as CottontailConnection) } \ No newline at end of file diff --git a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/vector/VectorDescriptorReader.kt b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/vector/VectorDescriptorReader.kt index 53508600..50157301 100644 --- a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/vector/VectorDescriptorReader.kt +++ b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/vector/VectorDescriptorReader.kt @@ -4,12 +4,13 @@ import org.vitrivr.cottontail.client.language.basics.Direction import org.vitrivr.cottontail.client.language.basics.Distances import org.vitrivr.cottontail.core.tuple.Tuple import org.vitrivr.engine.core.model.descriptor.vector.* +import org.vitrivr.engine.core.model.descriptor.vector.VectorDescriptor.Companion.VECTOR_ATTRIBUTE_NAME import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.query.Query import org.vitrivr.engine.core.model.query.proximity.ProximityQuery import org.vitrivr.engine.core.model.retrievable.Retrieved import org.vitrivr.engine.core.model.retrievable.attributes.DistanceAttribute -import org.vitrivr.engine.core.model.types.toValue +import org.vitrivr.engine.core.model.types.Value import org.vitrivr.engine.plugin.cottontaildb.* import org.vitrivr.engine.plugin.cottontaildb.descriptors.AbstractDescriptorReader @@ -17,7 +18,7 @@ import org.vitrivr.engine.plugin.cottontaildb.descriptors.AbstractDescriptorRead * An [AbstractDescriptorReader] for [FloatVectorDescriptor]s. * * @author Ralph Gasser - * @version 1.2.0 + * @version 1.3.1 */ internal class VectorDescriptorReader(field: Schema.Field<*, VectorDescriptor<*>>, connection: CottontailConnection) : AbstractDescriptorReader>(field, connection) { @@ -30,28 +31,7 @@ internal class VectorDescriptorReader(field: Schema.Field<*, VectorDescriptor<*> * @param query The [Query] to execute. */ override fun query(query: Query): Sequence> = when (query) { - is ProximityQuery<*> -> { - val cottontailQuery = org.vitrivr.cottontail.client.language.dql.Query(this.entityName) - .select(RETRIEVABLE_ID_COLUMN_NAME) - .distance( - DESCRIPTOR_COLUMN_NAME, - query.value.toCottontailValue(), - Distances.valueOf(query.distance.toString()), - DISTANCE_COLUMN_NAME - ) - .order(DISTANCE_COLUMN_NAME, Direction.valueOf(query.order.name)) - .limit(query.k) - - if (query.fetchVector) { - cottontailQuery.select(DESCRIPTOR_COLUMN_NAME) - cottontailQuery.select(DESCRIPTOR_ID_COLUMN_NAME) - } - - this.connection.client.query(cottontailQuery).asSequence().map { - tupleToDescriptor(it) - } - } - + is ProximityQuery<*> -> queryProximity(query) else -> throw UnsupportedOperationException("Query of typ ${query::class} is not supported by FloatVectorDescriptorReader.") } @@ -63,47 +43,9 @@ internal class VectorDescriptorReader(field: Schema.Field<*, VectorDescriptor<*> * @param query The [Query] that should be executed. * @return [Sequence] of [Retrieved]. */ - override fun queryAndJoin(query: Query): Sequence { - when (query) { - is ProximityQuery<*> -> { - val cottontailQuery = org.vitrivr.cottontail.client.language.dql.Query(this.entityName) - .select(DESCRIPTOR_COLUMN_NAME) - .select(DESCRIPTOR_ID_COLUMN_NAME) - .select(RETRIEVABLE_ID_COLUMN_NAME) - .distance( - DESCRIPTOR_COLUMN_NAME, - query.value.toCottontailValue(), - Distances.valueOf(query.distance.toString()), - DISTANCE_COLUMN_NAME - ) - .order(DISTANCE_COLUMN_NAME, Direction.valueOf(query.order.name)) - .limit(query.k) - - /* Fetch descriptors */ - val descriptors = this.connection.client.query(cottontailQuery).asSequence().map { tuple -> - val scoreIndex = tuple.indexOf(DISTANCE_COLUMN_NAME) - tupleToDescriptor(tuple) to if (scoreIndex > -1) { - tuple.asFloat(DISTANCE_COLUMN_NAME)?.let { DistanceAttribute(it) } - } else { - null - } - }.toList() - if (descriptors.isEmpty()) return emptySequence() - - /* Fetch retrievable ids. */ - val retrievables = this.fetchRetrievable(descriptors.mapNotNull { it.first.retrievableId }.toSet()) - return descriptors.asSequence().mapNotNull { descriptor -> - val retrievable = retrievables[descriptor.first.retrievableId] ?: return@mapNotNull null - - /* Append descriptor and distance attribute. */ - retrievable.addDescriptor(descriptor.first) - descriptor.second?.let { retrievable.addAttribute(it) } - retrievable - } - } - - else -> throw UnsupportedOperationException("Query of typ ${query::class} is not supported by FloatVectorDescriptorReader.") - } + override fun queryAndJoin(query: Query): Sequence = when (query) { + is ProximityQuery<*> -> queryAndJoinProximity(query) + else -> super.queryAndJoin(query) } /** @@ -114,44 +56,110 @@ internal class VectorDescriptorReader(field: Schema.Field<*, VectorDescriptor<*> */ override fun tupleToDescriptor(tuple: Tuple): VectorDescriptor<*> { val descriptorId = tuple.asUuidValue(DESCRIPTOR_ID_COLUMN_NAME)?.value - ?: throw IllegalArgumentException("The provided tuple is missing the required field '${DESCRIPTOR_ID_COLUMN_NAME}'.") + ?: throw IllegalArgumentException("The provided tuple is missing the required field '${DESCRIPTOR_ID_COLUMN_NAME}'.") val retrievableId = tuple.asUuidValue(RETRIEVABLE_ID_COLUMN_NAME)?.value - ?: throw IllegalArgumentException("The provided tuple is missing the required field '${RETRIEVABLE_ID_COLUMN_NAME}'.") + ?: throw IllegalArgumentException("The provided tuple is missing the required field '${RETRIEVABLE_ID_COLUMN_NAME}'.") return when (this.prototype) { is BooleanVectorDescriptor -> BooleanVectorDescriptor( descriptorId, retrievableId, - tuple.asBooleanVector(DESCRIPTOR_COLUMN_NAME)?.map { it.toValue() } - ?: throw IllegalArgumentException("The provided tuple is missing the required field '$DESCRIPTOR_COLUMN_NAME'.") + Value.BooleanVector(tuple.asBooleanVector(VECTOR_ATTRIBUTE_NAME) ?: throw IllegalArgumentException("The provided tuple is missing the required field '$VECTOR_ATTRIBUTE_NAME'.")) ) is FloatVectorDescriptor -> FloatVectorDescriptor( descriptorId, retrievableId, - tuple.asFloatVector(DESCRIPTOR_COLUMN_NAME)?.map { it.toValue() } - ?: throw IllegalArgumentException("The provided tuple is missing the required field '$DESCRIPTOR_COLUMN_NAME'.") + Value.FloatVector(tuple.asFloatVector(VECTOR_ATTRIBUTE_NAME) ?: throw IllegalArgumentException("The provided tuple is missing the required field '$VECTOR_ATTRIBUTE_NAME'.")) ) is DoubleVectorDescriptor -> DoubleVectorDescriptor( descriptorId, retrievableId, - tuple.asDoubleVector(DESCRIPTOR_COLUMN_NAME)?.map { it.toValue() } - ?: throw IllegalArgumentException("The provided tuple is missing the required field '$DESCRIPTOR_COLUMN_NAME'.") + Value.DoubleVector(tuple.asDoubleVector(VECTOR_ATTRIBUTE_NAME) ?: throw IllegalArgumentException("The provided tuple is missing the required field '$VECTOR_ATTRIBUTE_NAME'.")) ) is IntVectorDescriptor -> IntVectorDescriptor( descriptorId, retrievableId, - tuple.asIntVector(DESCRIPTOR_COLUMN_NAME)?.map { it.toValue() } - ?: throw IllegalArgumentException("The provided tuple is missing the required field '$DESCRIPTOR_COLUMN_NAME'.") + Value.IntVector(tuple.asIntVector(VECTOR_ATTRIBUTE_NAME) ?: throw IllegalArgumentException("The provided tuple is missing the required field '$VECTOR_ATTRIBUTE_NAME'.")) ) is LongVectorDescriptor -> LongVectorDescriptor( descriptorId, retrievableId, - tuple.asLongVector(DESCRIPTOR_COLUMN_NAME)?.map { it.toValue() } - ?: throw IllegalArgumentException("The provided tuple is missing the required field '$DESCRIPTOR_COLUMN_NAME'.") + Value.LongVector(tuple.asLongVector(VECTOR_ATTRIBUTE_NAME) ?: throw IllegalArgumentException("The provided tuple is missing the required field '$VECTOR_ATTRIBUTE_NAME'.")) + ) + } + } + + /** + * Executes a [ProximityQuery] and returns a [Sequence] of [VectorDescriptor]s. + * + * @param query The [ProximityQuery] to execute. + * @return [Sequence] of [VectorDescriptor]s. + */ + private fun queryAndJoinProximity(query: ProximityQuery<*>): Sequence { + val cottontailQuery = org.vitrivr.cottontail.client.language.dql.Query(this.entityName) + .select(VECTOR_ATTRIBUTE_NAME) + .select(DESCRIPTOR_ID_COLUMN_NAME) + .select(RETRIEVABLE_ID_COLUMN_NAME) + .distance( + VECTOR_ATTRIBUTE_NAME, + query.value.toCottontailValue(), + Distances.valueOf(query.distance.toString()), + DISTANCE_COLUMN_NAME ) + .order(DISTANCE_COLUMN_NAME, Direction.valueOf(query.order.name)) + .limit(query.k) + + /* Fetch descriptors */ + val descriptors = this.connection.client.query(cottontailQuery).asSequence().map { tuple -> + val scoreIndex = tuple.indexOf(DISTANCE_COLUMN_NAME) + tupleToDescriptor(tuple) to if (scoreIndex > -1) { + tuple.asDouble(DISTANCE_COLUMN_NAME)?.let { DistanceAttribute(it.toFloat()) } + } else { + null + } + }.toList() + if (descriptors.isEmpty()) return emptySequence() + + /* Fetch retrievable ids. */ + val retrievables = this.fetchRetrievable(descriptors.mapNotNull { it.first.retrievableId }.toSet()) + return descriptors.asSequence().mapNotNull { descriptor -> + val retrievable = retrievables[descriptor.first.retrievableId] ?: return@mapNotNull null + + /* Append descriptor and distance attribute. */ + retrievable.addDescriptor(descriptor.first) + descriptor.second?.let { retrievable.addAttribute(it) } + retrievable + } + } + + /** + * Executes a [ProximityQuery] and returns a [Sequence] of [VectorDescriptor]s. + * + * @param query The [ProximityQuery] to execute. + * @return [Sequence] of [VectorDescriptor]s. + */ + private fun queryProximity(query: ProximityQuery<*>): Sequence> { + val cottontailQuery = org.vitrivr.cottontail.client.language.dql.Query(this.entityName) + .select(RETRIEVABLE_ID_COLUMN_NAME) + .distance( + VECTOR_ATTRIBUTE_NAME, + query.value.toCottontailValue(), + Distances.valueOf(query.distance.toString()), + DISTANCE_COLUMN_NAME + ) + .order(DISTANCE_COLUMN_NAME, Direction.valueOf(query.order.name)) + .limit(query.k) + + if (query.fetchVector) { + cottontailQuery.select(VECTOR_ATTRIBUTE_NAME) + cottontailQuery.select(DESCRIPTOR_ID_COLUMN_NAME) + } + + return this.connection.client.query(cottontailQuery).asSequence().map { + tupleToDescriptor(it) } } } \ No newline at end of file diff --git a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/vector/VectorDescriptorWriter.kt b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/vector/VectorDescriptorWriter.kt deleted file mode 100644 index 9512f3a2..00000000 --- a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/descriptors/vector/VectorDescriptorWriter.kt +++ /dev/null @@ -1,99 +0,0 @@ -package org.vitrivr.engine.plugin.cottontaildb.descriptors.vector - -import io.github.oshai.kotlinlogging.KLogger -import io.github.oshai.kotlinlogging.KotlinLogging -import io.grpc.StatusRuntimeException -import org.vitrivr.cottontail.client.language.basics.expression.Column -import org.vitrivr.cottontail.client.language.basics.expression.Literal -import org.vitrivr.cottontail.client.language.basics.predicate.Compare -import org.vitrivr.cottontail.client.language.dml.BatchInsert -import org.vitrivr.cottontail.client.language.dml.Insert -import org.vitrivr.cottontail.client.language.dml.Update -import org.vitrivr.cottontail.core.values.UuidValue -import org.vitrivr.engine.core.model.descriptor.vector.VectorDescriptor -import org.vitrivr.engine.core.model.metamodel.Schema -import org.vitrivr.engine.plugin.cottontaildb.* -import org.vitrivr.engine.plugin.cottontaildb.descriptors.AbstractDescriptorWriter - -private val logger: KLogger = KotlinLogging.logger {} - -/** - * An [AbstractDescriptorWriter] for [VectorDescriptor]s. - * - * @author Ralph Gasser - * @version 1.0.0 - */ -class VectorDescriptorWriter(field: Schema.Field<*, VectorDescriptor<*>>, connection: CottontailConnection) : AbstractDescriptorWriter>(field, connection) { - - /** - * Adds (writes) a single [VectorDescriptor] using this [VectorDescriptorWriter]. - * - * @param item The [VectorDescriptor] to write. - * @return True on success, false otherwise. - */ - override fun add(item: VectorDescriptor<*>): Boolean { - val insert = Insert(this.entityName).values( - DESCRIPTOR_ID_COLUMN_NAME to UuidValue(item.id.toString()), - RETRIEVABLE_ID_COLUMN_NAME to UuidValue(item.retrievableId ?: throw IllegalArgumentException("A vector descriptor must be associated with a retrievable ID.")), - DESCRIPTOR_COLUMN_NAME to item.toCottontailValue() - ) - return try { - this.connection.client.insert(insert).use { - it.hasNext() - } - } catch (e: StatusRuntimeException) { - logger.error(e) { "Failed to persist descriptor ${item.id} due to exception." } - false - } - } - - /** - * Adds (writes) a batch of [VectorDescriptor] using this [VectorDescriptorWriter]. - * - * @param items A [Iterable] of [VectorDescriptor]s to write. - * @return True on success, false otherwise. - */ - override fun addAll(items: Iterable>): Boolean { - /* Prepare insert query. */ - var size = 0 - val insert = BatchInsert(this.entityName).columns(DESCRIPTOR_ID_COLUMN_NAME, RETRIEVABLE_ID_COLUMN_NAME, DESCRIPTOR_COLUMN_NAME) - for (item in items) { - size += 1 - insert.values(UuidValue(item.id), UuidValue(item.retrievableId ?: throw IllegalArgumentException("A vector descriptor must be associated with a retrievable ID.")), item.toCottontailValue()) - } - - /* Insert values. */ - return try { - this.connection.client.insert(insert) - true - } catch (e: StatusRuntimeException) { - logger.error(e) { "Failed to persist $size descriptors due to exception." } - false - } - } - - /** - * Updates a specific [VectorDescriptor] using this [VectorDescriptorWriter]. - * - * @param item A [VectorDescriptor]s to update. - * @return True on success, false otherwise. - */ - override fun update(item: VectorDescriptor<*>): Boolean { - val update = Update(this.entityName).where( - Compare( - Column(this.entityName.column(DESCRIPTOR_ID_COLUMN_NAME)), - Compare.Operator.EQUAL, - Literal(UuidValue(item.id)) - ) - ).values(DESCRIPTOR_COLUMN_NAME to item.toCottontailValue()) - - /* Delete values. */ - return try { - this.connection.client.update(update) - true - } catch (e: StatusRuntimeException) { - logger.error(e) { "Failed to update descriptor due to exception." } - false - } - } -} \ No newline at end of file diff --git a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/retrievable/RetrievableInitializer.kt b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/retrievable/RetrievableInitializer.kt index 71e3f3ac..71b73260 100644 --- a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/retrievable/RetrievableInitializer.kt +++ b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/retrievable/RetrievableInitializer.kt @@ -1,7 +1,5 @@ package org.vitrivr.engine.plugin.cottontaildb.retrievable -import io.github.oshai.kotlinlogging.KLogger -import io.github.oshai.kotlinlogging.KotlinLogging import io.grpc.StatusRuntimeException import org.vitrivr.cottontail.client.language.ddl.* import org.vitrivr.cottontail.core.database.Name @@ -11,9 +9,6 @@ import org.vitrivr.engine.core.database.retrievable.RetrievableInitializer import org.vitrivr.engine.core.model.retrievable.Retrievable import org.vitrivr.engine.plugin.cottontaildb.* -/** Defines [KLogger] of the class. */ -private val logger: KLogger = KotlinLogging.logger {} - /** * A [RetrievableInitializer] implementation for Cottontail DB. * @@ -37,7 +32,7 @@ internal class RetrievableInitializer(private val connection: CottontailConnecti try { this.connection.client.create(createSchema) } catch (e: StatusRuntimeException) { - logger.error(e) { "Failed to initialize entity ${this.entityName} due to exception." } + LOGGER.error(e) { "Failed to initialize entity ${this.entityName} due to exception." } } try { @@ -55,7 +50,7 @@ internal class RetrievableInitializer(private val connection: CottontailConnecti .column(this.entityName.column(RETRIEVABLE_TYPE_COLUMN_NAME)) ) } catch (e: StatusRuntimeException) { - logger.error(e) { "Failed to initialize entity ${this.entityName} due to exception." } + LOGGER.error(e) { "Failed to initialize entity ${this.entityName} due to exception." } } try { @@ -82,7 +77,24 @@ internal class RetrievableInitializer(private val connection: CottontailConnecti .column(this.relationshipEntityName.column(PREDICATE_COLUMN_NAME)) ) } catch (e: StatusRuntimeException) { - logger.error(e) { "Failed to initialize entity ${this.entityName} due to exception." } + LOGGER.error(e) { "Failed to initialize entity ${this.entityName} due to exception." } + } + } + + /** + * De-initializes the entities that is used to store [Retrievable]s in Cottontail DB. + */ + override fun deinitialize() { + try { + this.connection.client.drop(DropEntity(this.entityName)) + } catch (e: StatusRuntimeException) { + LOGGER.error(e) { "Failed to initialize entity ${this.entityName} due to exception." } + } + + try { + this.connection.client.drop(DropEntity(this.relationshipEntityName)) + } catch (e: StatusRuntimeException) { + LOGGER.error(e) { "Failed to initialize entity '${this.entityName}' due to exception." } } } @@ -107,7 +119,7 @@ internal class RetrievableInitializer(private val connection: CottontailConnecti try { this.connection.client.truncate(truncate) } catch (e: StatusRuntimeException) { - logger.error(e) { "Failed to truncate entity ${this.entityName} due to exception." } + LOGGER.error(e) { "Failed to truncate entity '${this.entityName}' due to exception." } } } } \ No newline at end of file diff --git a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/retrievable/RetrievableReader.kt b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/retrievable/RetrievableReader.kt index 2cee1e9b..45bba869 100644 --- a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/retrievable/RetrievableReader.kt +++ b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/retrievable/RetrievableReader.kt @@ -28,7 +28,7 @@ private val logger: KLogger = KotlinLogging.logger {} * @author Ralph Gasser * @version 1.0.0 */ -internal class RetrievableReader(private val connection: CottontailConnection) : RetrievableReader { +internal class RetrievableReader(override val connection: CottontailConnection) : RetrievableReader { /** The [Name.EntityName] for this [RetrievableInitializer]. */ private val entityName: Name.EntityName = Name.EntityName.create(this.connection.schemaName, RETRIEVABLE_ENTITY_NAME) diff --git a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/retrievable/RetrievableWriter.kt b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/retrievable/RetrievableWriter.kt index fffc53d0..c8145ab1 100644 --- a/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/retrievable/RetrievableWriter.kt +++ b/vitrivr-engine-module-cottontaildb/src/main/kotlin/org/vitrivr/engine/plugin/cottontaildb/retrievable/RetrievableWriter.kt @@ -28,7 +28,7 @@ private val logger: KLogger = KotlinLogging.logger {} * @author Ralph Gasser * @version 1.0.0 */ -internal class RetrievableWriter(private val connection: CottontailConnection) : RetrievableWriter { +internal class RetrievableWriter(override val connection: CottontailConnection) : RetrievableWriter { /** The [Name.EntityName] of the retrievable entity. */ private val entityName: Name.EntityName = Name.EntityName.create(this.connection.schemaName, RETRIEVABLE_ENTITY_NAME) @@ -122,8 +122,8 @@ internal class RetrievableWriter(private val connection: CottontailConnection) : /* Delete values. */ return try { - this.connection.client.delete(delete) - true + val result = this.connection.client.delete(delete) + return result.hasNext() && result.next().asLong(0)!! > 0L } catch (e: StatusRuntimeException) { logger.error(e) { "Failed to delete retrievable due to exception." } false diff --git a/vitrivr-engine-module-cottontaildb/src/test/kotlin/org/vitrivr/engine/database/cottontaildb/descriptor/DescriptorInitializerTest.kt b/vitrivr-engine-module-cottontaildb/src/test/kotlin/org/vitrivr/engine/database/cottontaildb/descriptor/DescriptorInitializerTest.kt new file mode 100644 index 00000000..b62c7b01 --- /dev/null +++ b/vitrivr-engine-module-cottontaildb/src/test/kotlin/org/vitrivr/engine/database/cottontaildb/descriptor/DescriptorInitializerTest.kt @@ -0,0 +1,11 @@ +package org.vitrivr.engine.database.cottontaildb.descriptor + +import org.vitrivr.engine.core.database.descriptor.AbstractDescriptorInitializerTest + +/** + * An [AbstractDescriptorInitializerTest] for Cottontail DB. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class DescriptorInitializerTest : AbstractDescriptorInitializerTest("test-schema-cottontaildb.json") \ No newline at end of file diff --git a/vitrivr-engine-module-cottontaildb/src/test/kotlin/org/vitrivr/engine/database/cottontaildb/descriptor/struct/FileMetadataDescriptorReaderTest.kt b/vitrivr-engine-module-cottontaildb/src/test/kotlin/org/vitrivr/engine/database/cottontaildb/descriptor/struct/FileMetadataDescriptorReaderTest.kt new file mode 100644 index 00000000..e0ceecdd --- /dev/null +++ b/vitrivr-engine-module-cottontaildb/src/test/kotlin/org/vitrivr/engine/database/cottontaildb/descriptor/struct/FileMetadataDescriptorReaderTest.kt @@ -0,0 +1,11 @@ +package org.vitrivr.engine.database.cottontaildb.descriptor.struct + +import org.vitrivr.engine.core.database.descriptor.struct.AbstractFileMetadataDescriptorReaderTest + +/** + * An [AbstractFileMetadataDescriptorReaderTest] implementation for PostgreSQL with pgVector. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class FileMetadataDescriptorReaderTest : AbstractFileMetadataDescriptorReaderTest("test-schema-cottontaildb.json") \ No newline at end of file diff --git a/vitrivr-engine-module-cottontaildb/src/test/kotlin/org/vitrivr/engine/database/cottontaildb/descriptor/vector/FloatVectorDescriptorReaderTest.kt b/vitrivr-engine-module-cottontaildb/src/test/kotlin/org/vitrivr/engine/database/cottontaildb/descriptor/vector/FloatVectorDescriptorReaderTest.kt new file mode 100644 index 00000000..bec913ea --- /dev/null +++ b/vitrivr-engine-module-cottontaildb/src/test/kotlin/org/vitrivr/engine/database/cottontaildb/descriptor/vector/FloatVectorDescriptorReaderTest.kt @@ -0,0 +1,11 @@ +package org.vitrivr.engine.database.cottontaildb.descriptor.vector + +import org.vitrivr.engine.core.database.descriptor.vector.AbstractFloatVectorDescriptorReaderTest + +/** + * An [AbstractFloatVectorDescriptorReaderTest] for Cottontail DB. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class FloatVectorDescriptorReaderTest : AbstractFloatVectorDescriptorReaderTest("test-schema-cottontaildb.json") \ No newline at end of file diff --git a/vitrivr-engine-module-cottontaildb/src/test/kotlin/org/vitrivr/engine/database/cottontaildb/retrievable/RetrievableInitializerTest.kt b/vitrivr-engine-module-cottontaildb/src/test/kotlin/org/vitrivr/engine/database/cottontaildb/retrievable/RetrievableInitializerTest.kt new file mode 100644 index 00000000..97362cc0 --- /dev/null +++ b/vitrivr-engine-module-cottontaildb/src/test/kotlin/org/vitrivr/engine/database/cottontaildb/retrievable/RetrievableInitializerTest.kt @@ -0,0 +1,11 @@ +package org.vitrivr.engine.database.cottontaildb.retrievable + +import org.vitrivr.engine.core.database.retrievable.AbstractRetrievableInitializerTest + +/** + * An [AbstractRetrievableInitializerTest] implementation for Cottontail DB. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class RetrievableInitializerTest : AbstractRetrievableInitializerTest("test-schema-cottontaildb.json") \ No newline at end of file diff --git a/vitrivr-engine-module-cottontaildb/src/test/kotlin/org/vitrivr/engine/database/cottontaildb/retrievable/RetrievableWriterTest.kt b/vitrivr-engine-module-cottontaildb/src/test/kotlin/org/vitrivr/engine/database/cottontaildb/retrievable/RetrievableWriterTest.kt new file mode 100644 index 00000000..c4385483 --- /dev/null +++ b/vitrivr-engine-module-cottontaildb/src/test/kotlin/org/vitrivr/engine/database/cottontaildb/retrievable/RetrievableWriterTest.kt @@ -0,0 +1,11 @@ +package org.vitrivr.engine.database.cottontaildb.retrievable + +import org.vitrivr.engine.core.database.retrievable.AbstractRetrievableWriterTest + +/** + * An [AbstractRetrievableWriterTest] for Cottontail DB. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class RetrievableWriterTest : AbstractRetrievableWriterTest("test-schema-cottontaildb.json") \ No newline at end of file diff --git a/vitrivr-engine-module-cottontaildb/src/test/resources/test-schema-cottontaildb.json b/vitrivr-engine-module-cottontaildb/src/test/resources/test-schema-cottontaildb.json new file mode 100644 index 00000000..f39f59b6 --- /dev/null +++ b/vitrivr-engine-module-cottontaildb/src/test/resources/test-schema-cottontaildb.json @@ -0,0 +1,43 @@ +{ + "name": "vitrivr-test", + "connection": { + "database": "CottontailConnectionProvider", + "parameters": { + "host": "127.0.0.1", + "port": "1865" + } + }, + "fields": [ + { + "name": "averagecolor", + "factory": "AverageColor" + }, + { + "name": "file", + "factory": "FileSourceMetadata", + "indexes": [ + { + "attributes": [ + "path" + ], + "type": "FULLTEXT" + }, + { + "attributes": [ + "size" + ], + "type": "SCALAR" + } + ] + }, + { + "name": "time", + "factory": "TemporalMetadata" + }, + { + "name": "video", + "factory": "VideoSourceMetadata" + } + ], + "exporters": [] +} diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/database/string/StringConnection.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/database/string/StringConnection.kt index 1299e3da..816f80df 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/database/string/StringConnection.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/database/string/StringConnection.kt @@ -1,19 +1,32 @@ package org.vitrivr.engine.module.features.database.string -import org.vitrivr.engine.module.features.database.string.writer.StringRetrievableWriter +import io.github.oshai.kotlinlogging.KLogger +import io.github.oshai.kotlinlogging.KotlinLogging.logger import org.vitrivr.engine.core.database.AbstractConnection import org.vitrivr.engine.core.database.retrievable.NoRetrievableInitializer import org.vitrivr.engine.core.database.retrievable.RetrievableInitializer import org.vitrivr.engine.core.database.retrievable.RetrievableReader import org.vitrivr.engine.core.database.retrievable.RetrievableWriter import org.vitrivr.engine.core.model.Persistable +import org.vitrivr.engine.module.features.database.string.writer.StringRetrievableWriter + + +/** Defines [KLogger] of the class. */ +internal val LOGGER: KLogger = logger("org.vitrivr.engine.module.database.string.StringConnection") class StringConnection(override val provider: StringConnectionProvider, schemaName: String, internal val stringify: (Persistable) -> String) : AbstractConnection(schemaName, provider) { + /** + * [StringConnection] does not support transactions. + */ + override fun withTransaction(action: (Unit) -> T): T { + LOGGER.warn { "Transactions are not supported by the StringConnection." } + return action.invoke(Unit) + } override fun getRetrievableInitializer(): RetrievableInitializer = NoRetrievableInitializer() - override fun getRetrievableWriter(): RetrievableWriter = StringRetrievableWriter(provider.targetStream, stringify) + override fun getRetrievableWriter(): RetrievableWriter = StringRetrievableWriter(this, provider.targetStream, stringify) override fun getRetrievableReader(): RetrievableReader { TODO("Not yet implemented") diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/database/string/writer/StringDescriptorWriter.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/database/string/writer/StringDescriptorWriter.kt index 87be81de..66c63105 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/database/string/writer/StringDescriptorWriter.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/database/string/writer/StringDescriptorWriter.kt @@ -1,9 +1,8 @@ package org.vitrivr.engine.module.features.database.string.writer -import org.vitrivr.engine.module.features.database.string.StringConnection import org.vitrivr.engine.core.database.descriptor.DescriptorWriter import org.vitrivr.engine.core.model.descriptor.Descriptor import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.module.features.database.string.StringConnection -class StringDescriptorWriter(connection: StringConnection, override val field: Schema.Field<*,D>) : DescriptorWriter, StringWriter(connection.provider.targetStream, connection.stringify) { -} \ No newline at end of file +class StringDescriptorWriter(connection: StringConnection, override val field: Schema.Field<*,D>) : DescriptorWriter, StringWriter(connection, connection.provider.targetStream, connection.stringify) \ No newline at end of file diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/database/string/writer/StringRetrievableWriter.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/database/string/writer/StringRetrievableWriter.kt index 21c4d5e3..4dbed8d2 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/database/string/writer/StringRetrievableWriter.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/database/string/writer/StringRetrievableWriter.kt @@ -4,9 +4,10 @@ import org.vitrivr.engine.core.database.retrievable.RetrievableWriter import org.vitrivr.engine.core.model.Persistable import org.vitrivr.engine.core.model.relationship.Relationship import org.vitrivr.engine.core.model.retrievable.Retrievable +import org.vitrivr.engine.module.features.database.string.StringConnection import java.io.OutputStream -class StringRetrievableWriter(outputStream: OutputStream, stringify: (Persistable) -> String) : StringWriter(outputStream, stringify), RetrievableWriter { +class StringRetrievableWriter(connection: StringConnection, outputStream: OutputStream, stringify: (Persistable) -> String) : StringWriter(connection, outputStream, stringify), RetrievableWriter { override fun connect(relationship: Relationship): Boolean { TODO("Not yet implemented") } diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/database/string/writer/StringWriter.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/database/string/writer/StringWriter.kt index 4e8d7ab9..3dea5891 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/database/string/writer/StringWriter.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/database/string/writer/StringWriter.kt @@ -2,11 +2,12 @@ package org.vitrivr.engine.module.features.database.string.writer import org.vitrivr.engine.core.database.Writer import org.vitrivr.engine.core.model.Persistable +import org.vitrivr.engine.module.features.database.string.StringConnection import java.io.IOException import java.io.OutputStream import java.io.PrintWriter -open class StringWriter(outputStream: OutputStream, private val stringify: (Persistable) -> String) : Writer { +open class StringWriter(override val connection: StringConnection, outputStream: OutputStream, private val stringify: (Persistable) -> String) : Writer { private val writer = PrintWriter(outputStream) diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/AverageColorRaster.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/averagecolorraster/AverageColorRaster.kt similarity index 86% rename from vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/AverageColorRaster.kt rename to vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/averagecolorraster/AverageColorRaster.kt index c7c58cd1..e20dc179 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/AverageColorRaster.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/averagecolorraster/AverageColorRaster.kt @@ -1,9 +1,8 @@ -package org.vitrivr.engine.module.features.feature.migration +package org.vitrivr.engine.module.features.feature.averagecolorraster import org.vitrivr.engine.core.context.IndexContext import org.vitrivr.engine.core.context.QueryContext import org.vitrivr.engine.core.model.content.element.ContentElement -import org.vitrivr.engine.core.model.descriptor.struct.RasterDescriptor import org.vitrivr.engine.core.model.metamodel.Analyser import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.query.Query @@ -18,7 +17,7 @@ class AverageColorRaster: Analyser, RasterDescriptor> { override val contentClasses = setOf(ContentElement::class) override val descriptorClass = RasterDescriptor::class override fun prototype(field: Schema.Field<*, *>): RasterDescriptor = - RasterDescriptor(id = UUID.randomUUID(), retrievableId = UUID.randomUUID(), hist = List(15) { Value.Float(0.0f) }, raster = List(64) { Value.Float(0.0f) }) // should transient be false? what is transient? + RasterDescriptor(id = UUID.randomUUID(), retrievableId = UUID.randomUUID(), mapOf("hist" to Value.FloatVector(FloatArray(15)), "raster" to Value.FloatVector(FloatArray(64)))) override fun newRetrieverForContent(field: Schema.Field, RasterDescriptor>, content: Collection>, context: QueryContext): Retriever, RasterDescriptor> { TODO("Not yet implemented") diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/averagecolorraster/RasterDescriptor.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/averagecolorraster/RasterDescriptor.kt new file mode 100644 index 00000000..486919e8 --- /dev/null +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/averagecolorraster/RasterDescriptor.kt @@ -0,0 +1,37 @@ +package org.vitrivr.engine.module.features.feature.averagecolorraster + +import org.vitrivr.engine.core.model.descriptor.DescriptorId +import org.vitrivr.engine.core.model.descriptor.Attribute +import org.vitrivr.engine.core.model.descriptor.AttributeName +import org.vitrivr.engine.core.model.descriptor.struct.MapStructDescriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.retrievable.RetrievableId +import org.vitrivr.engine.core.model.types.Type +import org.vitrivr.engine.core.model.types.Value + +/** + * A struct vector descriptor that stores the average color and raster of an image. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class RasterDescriptor( + override var id: DescriptorId, + override var retrievableId: RetrievableId?, + values: Map?>, + override val field: Schema.Field<*, RasterDescriptor>? = null +) : MapStructDescriptor(id, retrievableId, LAYOUT, values, field) { + + companion object { + private val LAYOUT = listOf( + Attribute("hist", Type.FloatVector(15)), + Attribute("raster", Type.FloatVector(4)) + ) + } + + /** The histogram vector. */ + val hist: Value.FloatVector by this.values + + /** The raster vector. */ + val raster: Value.FloatVector by this.values +} \ No newline at end of file diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/external/ExternalAnalyser.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/external/ExternalAnalyser.kt index 30828d28..686ad68c 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/external/ExternalAnalyser.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/external/ExternalAnalyser.kt @@ -83,7 +83,7 @@ abstract class ExternalAnalyser, U : Descriptor> : Analyse if (responseCode != HttpURLConnection.HTTP_OK) return null return connection.inputStream.use { stream -> when (U::class) { - FloatVectorDescriptor::class -> FloatVectorDescriptor(UUID.randomUUID(), null, Json.decodeFromStream(stream).map { Value.Float(it) }) + FloatVectorDescriptor::class -> FloatVectorDescriptor(UUID.randomUUID(), null, Value.FloatVector(Json.decodeFromStream(stream))) else -> null } as U? } diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/external/implementations/clip/CLIP.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/external/implementations/clip/CLIP.kt index fe1995e3..247b3984 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/external/implementations/clip/CLIP.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/external/implementations/clip/CLIP.kt @@ -27,7 +27,7 @@ import java.util.* * Implementation of the [CLIP] [ExternalAnalyser], which derives the CLIP feature from an [ImageContent] or [TextContent] as [FloatVectorDescriptor]. * * @author Rahel Arnold - * @version 1.2.0 + * @version 1.3.0 */ class CLIP : ExternalAnalyser, FloatVectorDescriptor>() { @@ -64,7 +64,7 @@ class CLIP : ExternalAnalyser, FloatVectorDescriptor>() { * * @return [FloatVectorDescriptor] */ - override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), List(512) { Value.Float(0.0f) }) + override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.FloatVector(512)) /** * Generates and returns a new [Extractor] instance for this [CLIP]. @@ -108,9 +108,9 @@ class CLIP : ExternalAnalyser, FloatVectorDescriptor>() { */ override fun newRetrieverForQuery(field: Schema.Field, FloatVectorDescriptor>, query: Query, context: QueryContext): DenseRetriever> { require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } - require(query is ProximityQuery<*> && query.value.first() is Value.Float) { "The query is not a ProximityQuery." } + require(query is ProximityQuery<*> && query.value is Value.FloatVector) { "The query is not a ProximityQuery." } @Suppress("UNCHECKED_CAST") - return DenseRetriever(field, query as ProximityQuery, context) + return DenseRetriever(field, query as ProximityQuery, context) } /** diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/external/implementations/dino/DINO.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/external/implementations/dino/DINO.kt index 65e78566..1473f344 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/external/implementations/dino/DINO.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/external/implementations/dino/DINO.kt @@ -55,7 +55,7 @@ class DINO : ExternalAnalyser() { * * @return [FloatVectorDescriptor] */ - override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), List(384) { Value.Float(0.0f) }) + override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.FloatVector(384)) /** * Generates and returns a new [Extractor] instance for this [DINO]. @@ -99,9 +99,9 @@ class DINO : ExternalAnalyser() { */ override fun newRetrieverForQuery(field: Schema.Field, query: Query, context: QueryContext): DenseRetriever { require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } - require(query is ProximityQuery<*> && query.value.first() is Value.Float) { "The query is not a ProximityQuery." } + require(query is ProximityQuery<*> && query.value is Value.FloatVector) { "The query is not a ProximityQuery." } @Suppress("UNCHECKED_CAST") - return DenseRetriever(field, query as ProximityQuery, context) + return DenseRetriever(field, query as ProximityQuery, context) } /** diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/AverageColorGrid8.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/AverageColorGrid8.kt index c719d255..d83590ec 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/AverageColorGrid8.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/AverageColorGrid8.kt @@ -17,7 +17,7 @@ import java.util.* class AverageColorGrid8 : Analyser, FloatVectorDescriptor> { override val contentClasses = setOf(ContentElement::class) override val descriptorClass = FloatVectorDescriptor::class - override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), List(192) { Value.Float(0.0f) }) + override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.FloatVector(192)) override fun newRetrieverForContent(field: Schema.Field, FloatVectorDescriptor>, content: Collection>, context: QueryContext): Retriever, FloatVectorDescriptor> { TODO("Not yet implemented") diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/AverageColorGrid8Reduced15.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/AverageColorGrid8Reduced15.kt index 6d1a2ea7..f0779e2e 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/AverageColorGrid8Reduced15.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/AverageColorGrid8Reduced15.kt @@ -17,7 +17,7 @@ import java.util.* class AverageColorGrid8Reduced15 : Analyser, FloatVectorDescriptor> { override val contentClasses = setOf(ContentElement::class) override val descriptorClass = FloatVectorDescriptor::class - override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), List(192) { Value.Float(0.0f) }) + override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.FloatVector(192)) override fun newRetrieverForContent(field: Schema.Field, FloatVectorDescriptor>, content: Collection>, context: QueryContext): Retriever, FloatVectorDescriptor> { TODO("Not yet implemented") diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/AverageFuzzyHist.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/AverageFuzzyHist.kt index 65fff5ef..765f50fa 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/AverageFuzzyHist.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/AverageFuzzyHist.kt @@ -17,7 +17,7 @@ import java.util.* class AverageFuzzyHist : Analyser, FloatVectorDescriptor> { override val contentClasses = setOf(ContentElement::class) override val descriptorClass = FloatVectorDescriptor::class - override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), List(15) { Value.Float(0.0f) }) + override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.FloatVector(15)) override fun newRetrieverForContent(field: Schema.Field, FloatVectorDescriptor>, content: Collection>, context: QueryContext): Retriever, FloatVectorDescriptor> { TODO("Not yet implemented") diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/AverageFuzzyHistNormalized.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/AverageFuzzyHistNormalized.kt index 5bf8dccf..e2ee9aec 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/AverageFuzzyHistNormalized.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/AverageFuzzyHistNormalized.kt @@ -17,7 +17,7 @@ import java.util.* class AverageFuzzyHistNormalized : Analyser, FloatVectorDescriptor> { override val contentClasses = setOf(ContentElement::class) override val descriptorClass = FloatVectorDescriptor::class - override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), List(15) { Value.Float(0.0f) }) + override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.FloatVector(15)) override fun newRetrieverForContent(field: Schema.Field, FloatVectorDescriptor>, content: Collection>, context: QueryContext): Retriever, FloatVectorDescriptor> { TODO("Not yet implemented") diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/CLD.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/CLD.kt index 88ee2dcb..caf1090f 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/CLD.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/CLD.kt @@ -17,7 +17,7 @@ import java.util.* class CLD : Analyser, FloatVectorDescriptor> { override val contentClasses = setOf(ContentElement::class) override val descriptorClass = FloatVectorDescriptor::class - override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), List(12) { Value.Float(0.0f) }) + override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.FloatVector(12)) override fun newRetrieverForContent(field: Schema.Field, FloatVectorDescriptor>, content: Collection>, context: QueryContext): Retriever, FloatVectorDescriptor> { TODO("Not yet implemented") diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/CLDReduced15.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/CLDReduced15.kt index 223d36ad..e28484d9 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/CLDReduced15.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/CLDReduced15.kt @@ -17,7 +17,7 @@ import java.util.* class CLDReduced15 : Analyser, FloatVectorDescriptor> { override val contentClasses = setOf(ContentElement::class) override val descriptorClass = FloatVectorDescriptor::class - override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), List(12) { Value.Float(0.0f) }) + override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.FloatVector(12)) override fun newRetrieverForContent(field: Schema.Field, FloatVectorDescriptor>, content: Collection>, context: QueryContext): Retriever, FloatVectorDescriptor> { TODO("Not yet implemented") diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/ConceptMasksADE20k.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/ConceptMasksADE20k.kt index 86e0f1b6..d8ea266f 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/ConceptMasksADE20k.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/ConceptMasksADE20k.kt @@ -17,7 +17,7 @@ import java.util.* class ConceptMasksADE20k : Analyser, FloatVectorDescriptor> { override val contentClasses = setOf(ContentElement::class) override val descriptorClass = FloatVectorDescriptor::class - override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), List(2048) { Value.Float(0.0f) }) + override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.FloatVector(2048)) override fun newRetrieverForContent(field: Schema.Field, FloatVectorDescriptor>, content: Collection>, context: QueryContext): Retriever, FloatVectorDescriptor> { TODO("Not yet implemented") diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/DominantEdgeGrid.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/DominantEdgeGrid.kt index c2160e8a..89c050aa 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/DominantEdgeGrid.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/DominantEdgeGrid.kt @@ -17,7 +17,7 @@ import java.util.* class DominantEdgeGrid : Analyser, FloatVectorDescriptor> { override val contentClasses = setOf(ContentElement::class) override val descriptorClass = FloatVectorDescriptor::class - override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), List(64) { Value.Float(0.0f) }) + override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.FloatVector(64)) override fun newRetrieverForContent(field: Schema.Field, FloatVectorDescriptor>, content: Collection>, context: QueryContext): Retriever, FloatVectorDescriptor> { TODO("Not yet implemented") diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/DominantEdgeGrid16.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/DominantEdgeGrid16.kt index dfed671a..f6cd65a9 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/DominantEdgeGrid16.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/DominantEdgeGrid16.kt @@ -17,7 +17,7 @@ import java.util.* class DominantEdgeGrid16 : Analyser, FloatVectorDescriptor> { override val contentClasses = setOf(ContentElement::class) override val descriptorClass = FloatVectorDescriptor::class - override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), List(256) { Value.Float(0.0f) }) + override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.FloatVector(256)) override fun newRetrieverForContent(field: Schema.Field, FloatVectorDescriptor>, content: Collection>, context: QueryContext): Retriever, FloatVectorDescriptor> { TODO("Not yet implemented") diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/DominantEdgeGrid8.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/DominantEdgeGrid8.kt index 7d113eec..6c50f83d 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/DominantEdgeGrid8.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/DominantEdgeGrid8.kt @@ -17,7 +17,7 @@ import java.util.* class DominantEdgeGrid8 : Analyser, FloatVectorDescriptor> { override val contentClasses = setOf(ContentElement::class) override val descriptorClass = FloatVectorDescriptor::class - override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), List(64) { Value.Float(0.0f) }) + override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.FloatVector(64)) override fun newRetrieverForContent(field: Schema.Field, FloatVectorDescriptor>, content: Collection>, context: QueryContext): Retriever, FloatVectorDescriptor> { TODO("Not yet implemented") diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/EHD.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/EHD.kt index cc6b7d0f..6e2ebad5 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/EHD.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/EHD.kt @@ -17,7 +17,7 @@ import java.util.* class EHD : Analyser, FloatVectorDescriptor> { override val contentClasses = setOf(ContentElement::class) override val descriptorClass = FloatVectorDescriptor::class - override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), List(80) { Value.Float(0.0f) }) + override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.FloatVector(80)) override fun newRetrieverForContent(field: Schema.Field, FloatVectorDescriptor>, content: Collection>, context: QueryContext): Retriever, FloatVectorDescriptor> { TODO("Not yet implemented") diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/EdgeARP88.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/EdgeARP88.kt index fffa3ee9..342328fb 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/EdgeARP88.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/EdgeARP88.kt @@ -17,7 +17,7 @@ import java.util.* class EdgeARP88 : Analyser, FloatVectorDescriptor> { override val contentClasses = setOf(ContentElement::class) override val descriptorClass = FloatVectorDescriptor::class - override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), List(64) { Value.Float(0.0f) }) + override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.FloatVector(64)) override fun newRetrieverForContent(field: Schema.Field, FloatVectorDescriptor>, content: Collection>, context: QueryContext): Retriever, FloatVectorDescriptor> { TODO("Not yet implemented") diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/EdgeGrid16.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/EdgeGrid16.kt index 130a9c27..8158b51f 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/EdgeGrid16.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/EdgeGrid16.kt @@ -17,7 +17,7 @@ import java.util.* class EdgeGrid16 : Analyser, FloatVectorDescriptor> { override val contentClasses = setOf(ContentElement::class) override val descriptorClass = FloatVectorDescriptor::class - override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), List(256) { Value.Float(0.0f) }) + override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.FloatVector(256)) override fun newRetrieverForContent(field: Schema.Field, FloatVectorDescriptor>, content: Collection>, context: QueryContext): Retriever, FloatVectorDescriptor> { TODO("Not yet implemented") diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/HOGMF25k512.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/HOGMF25k512.kt index bbd42660..f20686f7 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/HOGMF25k512.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/HOGMF25k512.kt @@ -17,7 +17,7 @@ import java.util.* class HOGMF25k512 : Analyser, FloatVectorDescriptor> { override val contentClasses = setOf(ContentElement::class) override val descriptorClass = FloatVectorDescriptor::class - override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), List(512) { Value.Float(0.0f) }) + override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.FloatVector(512)) override fun newRetrieverForContent(field: Schema.Field, FloatVectorDescriptor>, content: Collection>, context: QueryContext): Retriever, FloatVectorDescriptor> { TODO("Not yet implemented") diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/HueHistogram.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/HueHistogram.kt index 0b13a5f9..2cf63783 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/HueHistogram.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/HueHistogram.kt @@ -17,7 +17,7 @@ import java.util.* class HueHistogram : Analyser, FloatVectorDescriptor> { override val contentClasses = setOf(ContentElement::class) override val descriptorClass = FloatVectorDescriptor::class - override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), List(16) { Value.Float(0.0f) }) + override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.FloatVector(16)) override fun newRetrieverForContent(field: Schema.Field, FloatVectorDescriptor>, content: Collection>, context: QueryContext): Retriever, FloatVectorDescriptor> { TODO("Not yet implemented") diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/InceptionResNetV2.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/InceptionResNetV2.kt index 33c366c4..866516d3 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/InceptionResNetV2.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/InceptionResNetV2.kt @@ -17,7 +17,7 @@ import java.util.* class InceptionResNetV2 : Analyser, FloatVectorDescriptor> { override val contentClasses = setOf(ContentElement::class) override val descriptorClass = FloatVectorDescriptor::class - override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), List(1536) { Value.Float(0.0f) }) + override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.FloatVector(1536)) override fun newRetrieverForContent(field: Schema.Field, FloatVectorDescriptor>, content: Collection>, context: QueryContext): Retriever, FloatVectorDescriptor> { TODO("Not yet implemented") diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/MedianColor.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/MedianColor.kt index abfc77dd..19ed1bf9 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/MedianColor.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/MedianColor.kt @@ -17,7 +17,7 @@ import java.util.* class MedianColor : Analyser, FloatVectorDescriptor> { override val contentClasses = setOf(ContentElement::class) override val descriptorClass = FloatVectorDescriptor::class - override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), List(3) { Value.Float(0.0f) }) + override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.FloatVector(3)) override fun newRetrieverForContent(field: Schema.Field, FloatVectorDescriptor>, content: Collection>, context: QueryContext): Retriever, FloatVectorDescriptor> { TODO("Not yet implemented") diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/MedianFuzzyHist.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/MedianFuzzyHist.kt index bcec984f..6486f1a7 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/MedianFuzzyHist.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/MedianFuzzyHist.kt @@ -17,7 +17,7 @@ import java.util.* class MedianFuzzyHist : Analyser, FloatVectorDescriptor> { override val contentClasses = setOf(ContentElement::class) override val descriptorClass = FloatVectorDescriptor::class - override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), List(15) { Value.Float(0.0f) }) + override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.FloatVector(15)) override fun newRetrieverForContent(field: Schema.Field, FloatVectorDescriptor>, content: Collection>, context: QueryContext): Retriever, FloatVectorDescriptor> { TODO("Not yet implemented") diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/OpenCLIP.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/OpenCLIP.kt index cd355315..5edb0124 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/OpenCLIP.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/OpenCLIP.kt @@ -17,7 +17,7 @@ import java.util.* class OpenCLIP : Analyser, FloatVectorDescriptor> { override val contentClasses = setOf(ContentElement::class) override val descriptorClass = FloatVectorDescriptor::class - override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), List(512) { Value.Float(0.0f) }) + override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.FloatVector(512)) override fun newRetrieverForContent(field: Schema.Field, FloatVectorDescriptor>, content: Collection>, context: QueryContext): Retriever, FloatVectorDescriptor> { TODO("Not yet implemented") diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/SURFMF25K512.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/SURFMF25K512.kt index 2301a03b..b1367835 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/SURFMF25K512.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/SURFMF25K512.kt @@ -17,7 +17,7 @@ import java.util.* class SURFMF25K512 : Analyser, FloatVectorDescriptor> { override val contentClasses = setOf(ContentElement::class) override val descriptorClass = FloatVectorDescriptor::class - override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), List(512) { Value.Float(0.0f) }) + override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.FloatVector(512)) override fun newRetrieverForContent(field: Schema.Field, FloatVectorDescriptor>, content: Collection>, context: QueryContext): Retriever, FloatVectorDescriptor> { TODO("Not yet implemented") diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/VisualTextCoEmbedding.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/VisualTextCoEmbedding.kt index 02bcd4f7..eeced2b9 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/VisualTextCoEmbedding.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/VisualTextCoEmbedding.kt @@ -17,7 +17,7 @@ import java.util.* class VisualTextCoEmbedding : Analyser, FloatVectorDescriptor> { override val contentClasses = setOf(ContentElement::class) override val descriptorClass = FloatVectorDescriptor::class - override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), List(256) { Value.Float(0.0f) }) + override fun prototype(field: Schema.Field<*, *>) = FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.FloatVector(256)) override fun newRetrieverForContent(field: Schema.Field, FloatVectorDescriptor>, content: Collection>, context: QueryContext): Retriever, FloatVectorDescriptor> { TODO("Not yet implemented") diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/skeleton/SkeletonDescriptor.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/skeleton/SkeletonDescriptor.kt new file mode 100644 index 00000000..4a6231c7 --- /dev/null +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/skeleton/SkeletonDescriptor.kt @@ -0,0 +1,37 @@ +package org.vitrivr.engine.module.features.feature.skeleton + +import org.vitrivr.engine.core.model.descriptor.DescriptorId +import org.vitrivr.engine.core.model.descriptor.Attribute +import org.vitrivr.engine.core.model.descriptor.AttributeName +import org.vitrivr.engine.core.model.descriptor.struct.MapStructDescriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.retrievable.RetrievableId +import org.vitrivr.engine.core.model.types.Type +import org.vitrivr.engine.core.model.types.Value + +class SkeletonDescriptor( + override var id: DescriptorId, + override var retrievableId: RetrievableId?, + values: Map?>, + override val field: Schema.Field<*, SkeletonDescriptor>? = null +) : MapStructDescriptor(id, retrievableId, LAYOUT, values, field) { + + companion object { + private val LAYOUT = listOf( + Attribute("person", Type.Int), + Attribute("skeleton", Type.FloatVector(12)), + Attribute("weights", Type.FloatVector(12)) + ) + } + + /** The person index. */ + val person: Value.Int by this.values + + /** The vector describing the skeleton. */ + val skeleton: List by this.values + + /** The vector describing the skeleto weights. */ + val weights: List by this.values + + +} \ No newline at end of file diff --git a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/SkeletonPose.kt b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/skeleton/SkeletonPose.kt similarity index 84% rename from vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/SkeletonPose.kt rename to vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/skeleton/SkeletonPose.kt index 337d9d66..9d696cf3 100644 --- a/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/migration/SkeletonPose.kt +++ b/vitrivr-engine-module-features/src/main/kotlin/org/vitrivr/engine/module/features/feature/skeleton/SkeletonPose.kt @@ -1,9 +1,8 @@ -package org.vitrivr.engine.module.features.feature.migration +package org.vitrivr.engine.module.features.feature.skeleton import org.vitrivr.engine.core.context.IndexContext import org.vitrivr.engine.core.context.QueryContext import org.vitrivr.engine.core.model.content.element.ContentElement -import org.vitrivr.engine.core.model.descriptor.struct.SkeletonDescriptor import org.vitrivr.engine.core.model.metamodel.Analyser import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.query.Query @@ -13,16 +12,24 @@ import org.vitrivr.engine.core.operators.Operator import org.vitrivr.engine.core.operators.ingest.Extractor import org.vitrivr.engine.core.operators.retrieve.Retriever +/** + * A descriptor used to store a skeleton pose. + * + * @author Ralph Gasser + * @version 1.0.0 + */ class SkeletonPose : Analyser, SkeletonDescriptor> { override val contentClasses = setOf(ContentElement::class) override val descriptorClass = SkeletonDescriptor::class override fun prototype(field: Schema.Field<*, *>): SkeletonDescriptor = SkeletonDescriptor( id = java.util.UUID.randomUUID(), retrievableId = java.util.UUID.randomUUID(), - person = Value.Int(0), - skeleton = List(12) { Value.Float(0.0f) }, - weights = List(12) { Value.Float(0.0f) } - ) // should transient be false? what is transient? + mapOf( + "person" to Value.Int(0), + "skeleton" to Value.FloatVector(FloatArray(12)), + "weights" to Value.FloatVector(FloatArray(12)) + ) + ) override fun newRetrieverForContent(field: Schema.Field, SkeletonDescriptor>, content: Collection>, context: QueryContext): Retriever, SkeletonDescriptor> { TODO("Not yet implemented") diff --git a/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser b/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser index b7f83405..a8c5e73b 100644 --- a/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser +++ b/vitrivr-engine-module-features/src/main/resources/META-INF/services/org.vitrivr.engine.core.model.metamodel.Analyser @@ -1,10 +1,9 @@ -org.vitrivr.engine.module.features.feature.averagecolor.AverageColor org.vitrivr.engine.module.features.feature.external.implementations.dino.DINO org.vitrivr.engine.module.features.feature.external.implementations.clip.CLIP org.vitrivr.engine.module.features.feature.migration.WhisperASR org.vitrivr.engine.module.features.feature.migration.ProvidedOCR org.vitrivr.engine.module.features.feature.migration.AudioTranscription -org.vitrivr.engine.module.features.feature.migration.SkeletonPose +org.vitrivr.engine.module.features.feature.skeleton.SkeletonPose org.vitrivr.engine.module.features.feature.migration.MedianColor org.vitrivr.engine.module.features.feature.migration.CLD org.vitrivr.engine.module.features.feature.migration.CLDReduced15 @@ -24,7 +23,7 @@ org.vitrivr.engine.module.features.feature.migration.HOGMF25k512 org.vitrivr.engine.module.features.feature.migration.SURFMF25K512 org.vitrivr.engine.module.features.feature.migration.InceptionResNetV2 org.vitrivr.engine.module.features.feature.migration.ConceptMasksADE20k -org.vitrivr.engine.module.features.feature.migration.AverageColorRaster +org.vitrivr.engine.module.features.feature.averagecolorraster.AverageColorRaster org.vitrivr.engine.module.features.feature.migration.EdgeARP88 org.vitrivr.engine.module.features.feature.migration.DominantColor org.vitrivr.engine.module.features.feature.migration.DominantEdgeGrid8 \ No newline at end of file diff --git a/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/DenseEmbedding.kt b/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/DenseEmbedding.kt index d510689f..1ad33ea8 100644 --- a/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/DenseEmbedding.kt +++ b/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/DenseEmbedding.kt @@ -63,11 +63,11 @@ companion object { when (element) { is ImageContent -> { val index = imageContents.indexOf(element) - listOf(FloatVectorDescriptor(UUID.randomUUID(), null, imageResults!![index].map { Value.Float(it) }, null)) + listOf(FloatVectorDescriptor(UUID.randomUUID(), null, Value.FloatVector(imageResults!![index].toFloatArray()), null)) } is TextContent -> { val index = textContents.indexOf(element) - listOf(FloatVectorDescriptor(UUID.randomUUID(), null, textResults!![index].map { Value.Float(it) }, null)) + listOf(FloatVectorDescriptor(UUID.randomUUID(), null, Value.FloatVector(textResults!![index].toFloatArray()), null)) } else -> throw(IllegalArgumentException("Content type not supported")) } @@ -86,7 +86,7 @@ companion object { override fun prototype(field: Schema.Field<*, *>) : FloatVectorDescriptor { //convert to integer val length = field.parameters[LENGTH_PARAMETER_NAME]?.toIntOrNull() ?: LENGTH_PARAMETER_DEFAULT - return FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), List(length) { Value.Float(0.0f) }) + return FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.FloatVector(length)) } /** @@ -100,9 +100,9 @@ companion object { */ override fun newRetrieverForQuery(field: Schema.Field, FloatVectorDescriptor>, query: Query, context: QueryContext): Retriever, FloatVectorDescriptor> { require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } - require(query is ProximityQuery<*> && query.value.first() is Value.Float) { "The query is not a ProximityQuery." } + require(query is ProximityQuery<*> && query.value is Value.FloatVector) { "The query is not a ProximityQuery." } @Suppress("UNCHECKED_CAST") - return DenseRetriever(field, query as ProximityQuery, context) + return DenseRetriever(field, query as ProximityQuery, context) } /** diff --git a/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/ImageClassification.kt b/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/ImageClassification.kt index 34d5af6b..4d2f902b 100644 --- a/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/ImageClassification.kt +++ b/vitrivr-engine-module-fes/src/main/kotlin/org/vitrivr/engine/base/features/external/implementations/ImageClassification.kt @@ -52,7 +52,7 @@ class ImageClassification : ExternalFesAnalyser() for (result in results) { val filteredResults = result - .mapIndexed { index, score -> LabelDescriptor(UUID.randomUUID(), null, Value.String(classes[index]), Value.Float(score), null) } + .mapIndexed { index, score -> LabelDescriptor(UUID.randomUUID(), null, mapOf("label" to Value.String(classes[index]), "confidence" to Value.Float(score)), null) } .filter { it.confidence.value >= threshold } .sortedByDescending { it.confidence.value } .take(topk) @@ -74,7 +74,7 @@ class ImageClassification : ExternalFesAnalyser() * @return [LabelDescriptor] */ override fun prototype(field: Schema.Field<*,*>): LabelDescriptor { - return LabelDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.String(""), Value.Float(0.0f)) + return LabelDescriptor(UUID.randomUUID(), UUID.randomUUID(), mapOf("label" to Value.String(""), "confidence" to Value.Float(0.0f))) } override fun newRetrieverForContent(field: Schema.Field, content: Collection, context: QueryContext): Retriever { @@ -102,7 +102,7 @@ class ImageClassification : ExternalFesAnalyser() val batchSize = context.getProperty(name, BATCHSIZE_PARAMETER_NAME)?.toIntOrNull() ?: BATCHSIZE_PARAMETER_DEFAULT.toInt() return object : FesExtractor(input, null, batchSize) { override fun assignRetrievableId(descriptor: LabelDescriptor, retrievableId: RetrievableId): LabelDescriptor { - return descriptor.copy(retrievableId = retrievableId) + return LabelDescriptor(descriptor.id, retrievableId, descriptor.values(), descriptor.field) } } } @@ -124,7 +124,7 @@ class ImageClassification : ExternalFesAnalyser() val batchSize = context.getProperty(field.fieldName, BATCHSIZE_PARAMETER_NAME)?.toIntOrNull() ?: BATCHSIZE_PARAMETER_DEFAULT.toInt() return object : FesExtractor(input, field, batchSize) { override fun assignRetrievableId(descriptor: LabelDescriptor, retrievableId: RetrievableId): LabelDescriptor { - return descriptor.copy(retrievableId = retrievableId, field = field) + return LabelDescriptor(descriptor.id, retrievableId, descriptor.values(), field) } } } diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/ModelHandler.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/ModelHandler.kt index d322f20f..db1fd7bb 100644 --- a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/ModelHandler.kt +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/ModelHandler.kt @@ -6,7 +6,6 @@ import org.joml.Vector4f import org.lwjgl.BufferUtils import org.lwjgl.assimp.* import org.lwjgl.assimp.Assimp.* - import org.lwjgl.system.MemoryStack import org.vitrivr.engine.core.model.mesh.* import java.io.File @@ -15,7 +14,6 @@ import java.io.InputStream import java.nio.ByteBuffer import java.nio.IntBuffer import java.util.* -import kotlin.collections.ArrayList class ModelHandler { @@ -95,9 +93,6 @@ class ModelHandler { LOGGER.trace("Try loading model {} from InputStream", modelId) val aiScene = loadAIScene(modelId, inputStream) - - val modelDir = File("").absolutePath // You may need to adjust this to the appropriate directory - val materialList: MutableList = ArrayList() val defaultMaterial = Material(mutableListOf()) // Define a default material val aiMeshes = aiScene.mMeshes() diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/features/sphericalharmonics/SphericalHarmonics.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/features/sphericalharmonics/SphericalHarmonics.kt index f90c4137..e6bcd440 100644 --- a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/features/sphericalharmonics/SphericalHarmonics.kt +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/features/sphericalharmonics/SphericalHarmonics.kt @@ -94,12 +94,12 @@ class SphericalHarmonics: Analyser { val numberOfCoefficients: Int = SphericalHarmonicsFunction.numberOfCoefficients(maxL, true) - SphericalHarmonicsFunction.numberOfCoefficients(minL - 1, true) /* Prepares an empty array for the feature vector. */ - val feature = Array((R - cap) * numberOfCoefficients) { _ -> Value.Float(0f) } + val feature = FloatArray((R - cap) * numberOfCoefficients) { _ -> 0f } /* Voxelizes the grid from the mesh. If the resulting grid is invisible, the method returns immediately. */ val grid: VoxelModel = voxelizer.voxelize(mesh, gridSize + 1, gridSize + 1, gridSize + 1) if (!grid.isVisible()) { - return FloatVectorDescriptor(UUID.randomUUID(), null, feature.toList()) + return FloatVectorDescriptor(UUID.randomUUID(), null, Value.FloatVector(feature)) } val descriptors: MutableList> = LinkedList() @@ -144,12 +144,12 @@ class SphericalHarmonics: Analyser { var index = 0 for (radius in descriptors) { for (descriptor in radius) { - feature[index++] = Value.Float(descriptor.abs().toFloat()) + feature[index++] = descriptor.abs().toFloat() } } /* Returns the normalized vector. */ - return FloatVectorDescriptor(UUID.randomUUID(), null, MathHelper.normalizeL2(feature).toList()) + return FloatVectorDescriptor(UUID.randomUUID(), null, Value.FloatVector(MathHelper.normalizeL2(feature))) } } @@ -162,16 +162,16 @@ class SphericalHarmonics: Analyser { val maxL = field.parameters[MAXL_PARAMETER_NAME]?.toIntOrNull() ?: MAXL_PARAMETER_DEFAULT val numberOfCoefficients: Int = SphericalHarmonicsFunction.numberOfCoefficients(maxL, true) - SphericalHarmonicsFunction.numberOfCoefficients(minL - 1, true) val vectorSize = ((gridSize / 2) - cap) * numberOfCoefficients - return FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), List(vectorSize) { Value.Float(0f) }) + return FloatVectorDescriptor(UUID.randomUUID(), UUID.randomUUID(), Value.FloatVector(FloatArray(vectorSize))) } override fun newRetrieverForQuery(field: Schema.Field, query: Query, context: QueryContext): Retriever { require(field.analyser == this) { "The field '${field.fieldName}' analyser does not correspond with this analyser. This is a programmer's error!" } - require(query is ProximityQuery<*> && query.value.first() is Value.Float) { } + require(query is ProximityQuery<*> && query.value is Value.FloatVector) { } /* Construct query. */ @Suppress("UNCHECKED_CAST") - return SphericalHarmonicsRetriever(field, query as ProximityQuery, context) + return SphericalHarmonicsRetriever(field, query as ProximityQuery, context) } override fun newRetrieverForDescriptors(field: Schema.Field, descriptors: Collection, context: QueryContext): Retriever { diff --git a/vitrivr-engine-module-pgvector/build.gradle b/vitrivr-engine-module-pgvector/build.gradle new file mode 100644 index 00000000..44412dd3 --- /dev/null +++ b/vitrivr-engine-module-pgvector/build.gradle @@ -0,0 +1,65 @@ +plugins { + id 'maven-publish' + id 'signing' +} + +repositories { + mavenCentral() +} + +dependencies { + api project(':vitrivr-engine-core') + + /** PostgreSQL driver. */ + implementation group: 'org.postgresql', name: 'postgresql', version: version_jdbc_postgres + + /** Vitrivr engine Core is required for running tests. */ + testImplementation(testFixtures(project(':vitrivr-engine-core'))) +} + +/* Publication of vitrivr engine query to Maven Central. */ +publishing { + publications { + mavenJava(MavenPublication) { + groupId = 'org.vitrivr' + artifactId = 'vitrivr-engine-module-pgvector' + version = System.getenv().getOrDefault("MAVEN_PUBLICATION_VERSION", version.toString()) + from components.java + pom { + name = 'vitrivr Engine pgVector Plugin' + description = 'A plugin that adds connectivity to the PostgreSQL with pgVector database.' + url = 'https://github.com/vitrivr/vitrivr-engine/' + licenses { + license { + name = 'MIT License' + } + } + developers { + developer { + id = 'ppanopticon' + name = 'Ralph Gasser' + email = 'ralph.gasser@unibas.ch' + } + } + scm { + connection = 'scm:git:https://github.com/vitrivr/vitrivr-engine.git' + url = 'https://github.com/vitrivr/vitrivr-engine/' + } + } + } + } + repositories { + repositories { + maven { + def releasesRepoUrl = 'https://oss.sonatype.org/service/local/staging/deploy/maven2/' + def snapshotsRepoUrl = 'https://oss.sonatype.org/content/repositories/snapshots/' + name = "OSSRH" + url = version.endsWith('SNAPSHOT') ? snapshotsRepoUrl : releasesRepoUrl + credentials { + username = System.getenv("MAVEN_USERNAME") + password = System.getenv("MAVEN_PASSWORD") + } + } + } + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/Constants.kt b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/Constants.kt new file mode 100644 index 00000000..735a41cd --- /dev/null +++ b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/Constants.kt @@ -0,0 +1,31 @@ +package org.vitrivr.engine.database.pgvector + +/** The name of the retrievable entity. */ +const val RETRIEVABLE_ENTITY_NAME = "retrievable" + +/** The column name of a retrievable ID. */ +const val RETRIEVABLE_ID_COLUMN_NAME = "retrievableId" + +/** The column name of a retrievable ID. */ +const val RETRIEVABLE_TYPE_COLUMN_NAME = "type" + +/** The name of the retrievable entity. */ +const val RELATIONSHIP_ENTITY_NAME = "relationships" + +/** The column name of a retrievable ID. */ +const val SUBJECT_ID_COLUMN_NAME = "subjectId" + +/** The column name of a retrievable ID. */ +const val OBJECT_ID_COLUMN_NAME = "objectId" + +/** The column name of a retrievable ID. */ +const val PREDICATE_COLUMN_NAME = "predicate" + +/** The prefix for descriptor entities. */ +const val DESCRIPTOR_ENTITY_PREFIX = "descriptor" + +/** The column name of a descriptor ID. */ +const val DESCRIPTOR_ID_COLUMN_NAME = "descriptorId" + +/** The column name used to describe a distance.*/ +const val DISTANCE_COLUMN_NAME = "distance" \ No newline at end of file diff --git a/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/PgVectorConnection.kt b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/PgVectorConnection.kt new file mode 100644 index 00000000..1b256dc4 --- /dev/null +++ b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/PgVectorConnection.kt @@ -0,0 +1,126 @@ +package org.vitrivr.engine.database.pgvector + +import io.github.oshai.kotlinlogging.KLogger +import io.github.oshai.kotlinlogging.KotlinLogging.logger +import org.postgresql.PGConnection +import org.vitrivr.engine.core.database.AbstractConnection +import org.vitrivr.engine.core.database.retrievable.RetrievableInitializer +import org.vitrivr.engine.core.database.retrievable.RetrievableReader +import org.vitrivr.engine.core.database.retrievable.RetrievableWriter +import org.vitrivr.engine.database.pgvector.descriptor.model.PgBitVector +import org.vitrivr.engine.database.pgvector.descriptor.model.PgVector +import java.sql.Connection +import java.sql.SQLException + + +/** Defines [KLogger] of the class. */ +internal val LOGGER: KLogger = logger("org.vitrivr.engine.database.pgvector.PgVectorConnection") + +/** + * A [AbstractConnection] to connect to a PostgreSQL instance with the pgVector extension. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class PgVectorConnection(provider: PgVectorConnectionProvider, schemaName: String, val jdbc: Connection) : AbstractConnection(schemaName, provider) { + + init { + /* Make sure that the pg_vector extension is installed. */ + try { + this.jdbc.prepareStatement("CREATE EXTENSION IF NOT EXISTS vector;").use { + it.execute() + } + } catch (e: SQLException) { + LOGGER.error(e) { "Failed to create extension pg_vector due to exception." } + throw e + } + + /* Create necessary schema. */ + try { + this.jdbc.prepareStatement("CREATE SCHEMA \"${schemaName}\";").use { + it.execute() + } + } catch (e: SQLException) { + if (e.sqlState == "42P06") { + LOGGER.info { "Schema '$schemaName' already exists." } + } else { + LOGGER.error(e) { "Failed to create schema '$schemaName' due to exception." } + throw e + } + } + + try { + this.jdbc.prepareStatement("SET search_path TO \"$schemaName\", public;").use { + it.execute() + } + } catch (e: SQLException) { + LOGGER.error(e) { "Failed to set search path '$schemaName' due to exception." } + throw e + } + + /* Register the vector data type. */ + this.jdbc.unwrap(PGConnection::class.java).addDataType("vector", PgVector::class.java) + this.jdbc.unwrap(PGConnection::class.java).addDataType("bit", PgBitVector::class.java) + } + + /** + * Tries to execute a given action within a database transaction. + * + * @param action The action to execute within the transaction. + */ + @Synchronized + override fun withTransaction(action: (Unit) -> T): T { + try { + this.jdbc.autoCommit = false + val ret = action.invoke(Unit) + this.jdbc.commit() + return ret + } catch (e: Throwable) { + LOGGER.error(e) { "Failed to execute transaction due to exception." } + this.jdbc.rollback() + throw e + } finally { + this.jdbc.autoCommit = true + } + } + + /** + * Generates and returns a [RetrievableInitializer] for this [PgVectorConnection]. + * + * @return [RetrievableInitializer] + */ + override fun getRetrievableInitializer(): RetrievableInitializer + = org.vitrivr.engine.database.pgvector.retrievable.RetrievableInitializer(this) + + /** + * Generates and returns a [RetrievableWriter] for this [PgVectorConnection]. + * + * @return [RetrievableWriter] + */ + override fun getRetrievableWriter(): RetrievableWriter + = org.vitrivr.engine.database.pgvector.retrievable.RetrievableWriter(this) + + /** + * Generates and returns a [RetrievableWriter] for this [PgVectorConnection]. + * + * @return [RetrievableReader] + */ + override fun getRetrievableReader(): RetrievableReader + = org.vitrivr.engine.database.pgvector.retrievable.RetrievableReader(this) + + /** + * Returns the human-readable description of this [PgVectorConnection]. + */ + override fun description(): String = this.jdbc.toString() + + /** + * Closes this [PgVectorConnection] + */ + override fun close() { + try { + this.jdbc.close() + } catch (e: SQLException) { + LOGGER.warn(e) { "Failed to close database connection due to exception." } + } + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/PgVectorConnectionProvider.kt b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/PgVectorConnectionProvider.kt new file mode 100644 index 00000000..1ed12572 --- /dev/null +++ b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/PgVectorConnectionProvider.kt @@ -0,0 +1,119 @@ +package org.vitrivr.engine.database.pgvector + +import org.vitrivr.engine.core.database.AbstractConnectionProvider +import org.vitrivr.engine.core.database.Connection +import org.vitrivr.engine.core.database.ConnectionProvider +import org.vitrivr.engine.core.database.descriptor.DescriptorProvider +import org.vitrivr.engine.core.model.descriptor.scalar.* +import org.vitrivr.engine.core.model.descriptor.struct.LabelDescriptor +import org.vitrivr.engine.core.model.descriptor.struct.MapStructDescriptor +import org.vitrivr.engine.core.model.descriptor.struct.metadata.MediaDimensionsDescriptor +import org.vitrivr.engine.core.model.descriptor.struct.metadata.Rectangle2DMetadataDescriptor +import org.vitrivr.engine.core.model.descriptor.struct.metadata.TemporalMetadataDescriptor +import org.vitrivr.engine.core.model.descriptor.struct.metadata.source.FileSourceMetadataDescriptor +import org.vitrivr.engine.core.model.descriptor.struct.metadata.source.VideoSourceMetadataDescriptor +import org.vitrivr.engine.core.model.descriptor.vector.* +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.database.pgvector.descriptor.scalar.ScalarDescriptorProvider +import org.vitrivr.engine.database.pgvector.descriptor.struct.StructDescriptorProvider +import org.vitrivr.engine.database.pgvector.descriptor.vector.VectorDescriptorProvider +import java.sql.DriverManager +import java.util.* + + +/** + * Implementation of the [ConnectionProvider] interface for PostgreSQL with the PGVector extension. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class PgVectorConnectionProvider: AbstractConnectionProvider() { + + companion object { + /** Name of the host parameter. */ + const val PARAMETER_NAME_HOST = "host" + + /** Name of the host parameter. */ + const val PARAMETER_DEFAULT_HOST = "127.0.0.1" + + /** Name of the port parameter. */ + const val PARAMETER_NAME_PORT = "port" + + /** Name of the host parameter. */ + const val PARAMETER_DEFAULT_PORT = 5432 + + /** Name of the port parameter. */ + const val PARAMETER_NAME_DATABASE = "database" + + /** Name of the port parameter. */ + const val PARAMETER_DEFAULT_DATABASE = "postgres" + + /** Name of the host parameter. */ + const val PARAMETER_NAME_USERNAME = "username" + + /** Name of the host parameter. */ + const val PARAMETER_NAME_PASSWORD = "password" + + /** Name of the host parameter. */ + const val PARAMETER_NAME_SSL = "ssl" + + } + + /** The name of this [PgVectorConnectionProvider]. */ + override val databaseName: String = "PostgreSQL with pgVector" + + /** The version of this [PgVectorConnectionProvider]. */ + override val version: String = "1.0.0" + + /** + * This method is called during initialization of the [PgVectorConnectionProvider] and can be used to register [DescriptorProvider]s. + */ + override fun initialize() { + /* Scalar descriptors. */ + this.register(BooleanDescriptor::class, ScalarDescriptorProvider) + this.register(IntDescriptor::class, ScalarDescriptorProvider) + this.register(LongDescriptor::class, ScalarDescriptorProvider) + this.register(FloatDescriptor::class, ScalarDescriptorProvider) + this.register(DoubleDescriptor::class, ScalarDescriptorProvider) + this.register(StringDescriptor::class, ScalarDescriptorProvider) + + /* Vector descriptors. */ + this.register(BooleanVectorDescriptor::class, VectorDescriptorProvider) + this.register(IntVectorDescriptor::class, VectorDescriptorProvider) + this.register(LongVectorDescriptor::class, VectorDescriptorProvider) + this.register(FloatVectorDescriptor::class, VectorDescriptorProvider) + this.register(DoubleVectorDescriptor::class, VectorDescriptorProvider) + + /* Struct descriptor. */ + this.register(LabelDescriptor::class, StructDescriptorProvider) + this.register(FileSourceMetadataDescriptor::class, StructDescriptorProvider) + this.register(VideoSourceMetadataDescriptor::class, StructDescriptorProvider) + this.register(TemporalMetadataDescriptor::class, StructDescriptorProvider) + this.register(Rectangle2DMetadataDescriptor::class, StructDescriptorProvider) + this.register(MediaDimensionsDescriptor::class, StructDescriptorProvider) + this.register(MapStructDescriptor::class, StructDescriptorProvider) + } + + /** + * Opens a new [PgVectorConnection] for the given [Schema]. + * + * @param parameters The optional parameters. + * @return [Connection] + */ + override fun openConnection(schemaName: String, parameters: Map): Connection { + /* Prepare connection URL. */ + val host = parameters.getOrDefault(PARAMETER_NAME_HOST, PARAMETER_DEFAULT_HOST) + val port = parameters[PARAMETER_NAME_PORT]?.toInt() ?: PARAMETER_DEFAULT_PORT + val database = parameters[PARAMETER_NAME_DATABASE]?.toInt() ?: PARAMETER_DEFAULT_DATABASE + val url = "jdbc:postgresql://${host}:${port}/${database}" + + /* Prepare properties (optional). */ + val props = Properties() + parameters[PARAMETER_NAME_USERNAME]?.let { props.setProperty("user", it) } + parameters[PARAMETER_NAME_PASSWORD]?.let { props.setProperty("password", it) } + parameters[PARAMETER_NAME_SSL]?.let { props.setProperty("ssl", it) } + + /* Open JDBC connection and return PgVectorConnection. */ + return PgVectorConnection(this, schemaName, DriverManager.getConnection(url, props)) + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/Utilities.kt b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/Utilities.kt new file mode 100644 index 00000000..4c620286 --- /dev/null +++ b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/Utilities.kt @@ -0,0 +1,84 @@ +package org.vitrivr.engine.database.pgvector + +import org.vitrivr.engine.core.model.query.basics.ComparisonOperator +import org.vitrivr.engine.core.model.query.basics.Distance +import org.vitrivr.engine.core.model.query.basics.Distance.* +import org.vitrivr.engine.core.model.types.Type +import org.vitrivr.engine.core.model.types.Value +import org.vitrivr.engine.database.pgvector.descriptor.model.PgBitVector +import org.vitrivr.engine.database.pgvector.descriptor.model.PgVector +import java.sql.Date +import java.sql.JDBCType +import java.sql.PreparedStatement +import java.sql.SQLType + +/** + * Sets a value of [Value] type in a [PreparedStatement]. + * + * @param index The index to set. + * @param value The [Value] to set. + */ +internal fun PreparedStatement.setValue(index: Int, value: Value<*>) = when (value) { + is Value.Boolean -> this.setBoolean(index, value.value) + is Value.Byte -> this.setByte(index, value.value) + is Value.DateTime -> this.setDate(index, Date(value.value.toInstant().toEpochMilli())) + is Value.Double -> this.setDouble(index, value.value) + is Value.Float -> this.setFloat(index, value.value) + is Value.Int -> this.setInt(index, value.value) + is Value.Long -> this.setLong(index, value.value) + is Value.Short -> this.setShort(index, value.value) + is Value.String -> this.setString(index, value.value) + is Value.FloatVector -> this.setObject(index, PgVector(value.value)) + is Value.DoubleVector -> this.setObject(index, PgVector(value.value)) + is Value.IntVector -> this.setObject(index, PgVector(value.value)) + is Value.LongVector -> this.setObject(index, PgVector(value.value)) + is Value.BooleanVector -> this.setObject(index, PgBitVector(value.value)) + else -> throw IllegalArgumentException("Unsupported value type for vector value.") +} + +/** + * Converts a [Type] to a [SQLType]. + */ +internal fun Type.toSql(): Int = when (this) { + Type.Boolean -> JDBCType.BOOLEAN + Type.Byte -> JDBCType.TINYINT + Type.Short -> JDBCType.SMALLINT + Type.Int -> JDBCType.INTEGER + Type.Long -> JDBCType.BIGINT + Type.Float -> JDBCType.REAL + Type.Double -> JDBCType.DOUBLE + Type.Datetime -> JDBCType.DATE + Type.String -> JDBCType.VARCHAR + Type.Text -> JDBCType.CLOB + is Type.BooleanVector -> JDBCType.ARRAY + is Type.DoubleVector -> JDBCType.ARRAY + is Type.FloatVector -> JDBCType.ARRAY + is Type.IntVector -> JDBCType.ARRAY + is Type.LongVector -> JDBCType.ARRAY +}.ordinal + +/** + * Converts a [Distance] to a pgVector distance operator. + */ +fun Distance.toSql() = when(this) { + MANHATTAN -> "<+>" + EUCLIDEAN -> "<->" + COSINE -> "<=>" + HAMMING -> "<~>" + JACCARD -> "<%>" +} + +/** + * Converts a [ComparisonOperator] to a SQL operator. + * + * @return SQL comparison operator. + */ +internal fun ComparisonOperator.toSql(): String = when (this){ + ComparisonOperator.EQ -> "=" + ComparisonOperator.NEQ -> "!=" + ComparisonOperator.LE -> "<" + ComparisonOperator.GR -> ">" + ComparisonOperator.LEQ -> ">=" + ComparisonOperator.GEQ -> "<=" + ComparisonOperator.LIKE -> "LIKE" +} \ No newline at end of file diff --git a/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/AbstractDescriptorReader.kt b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/AbstractDescriptorReader.kt new file mode 100644 index 00000000..dddc0a71 --- /dev/null +++ b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/AbstractDescriptorReader.kt @@ -0,0 +1,225 @@ +package org.vitrivr.engine.database.pgvector.descriptor + +import org.vitrivr.engine.core.database.descriptor.DescriptorReader +import org.vitrivr.engine.core.model.descriptor.Descriptor +import org.vitrivr.engine.core.model.descriptor.DescriptorId +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.query.Query +import org.vitrivr.engine.core.model.retrievable.RetrievableId +import org.vitrivr.engine.core.model.retrievable.Retrieved +import org.vitrivr.engine.database.pgvector.* +import java.sql.ResultSet +import java.util.* + +/** + * An abstract implementation of a [DescriptorReader] for Cottontail DB. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +abstract class AbstractDescriptorReader(final override val field: Schema.Field<*, D>, override val connection: PgVectorConnection) : DescriptorReader { + + /** The name of the table backing this [AbstractDescriptorReader]. */ + protected val tableName: String = "${DESCRIPTOR_ENTITY_PREFIX}_${this.field.fieldName}" + + + /** The [Descriptor] prototype handled by this [AbstractDescriptorReader]. */ + protected val prototype: D= this.field.analyser.prototype(this.field) + + /** + * Returns a single [Descriptor]s of type [D] that has the provided [DescriptorId]. + * + * @param descriptorId The [DescriptorId], i.e., the [UUID] of the [Descriptor] to return. + * @return [Sequence] of all [Descriptor]s. + */ + override fun get(descriptorId: DescriptorId): D? { + try { + this.connection.jdbc.prepareStatement("SELECT * FROM $tableName WHERE $DESCRIPTOR_ID_COLUMN_NAME = ?").use { stmt -> + stmt.setObject(1, descriptorId) + stmt.executeQuery().use { res -> + if (res.next()) { + return this.rowToDescriptor(res) + } else { + return null + } + } + } + } catch (e: Exception) { + LOGGER.error(e) { "Failed to check fetch descriptor $descriptorId from '$tableName' due to SQL error." } + return null + } + } + + /** + * Returns the [Descriptor]s of type [D] that belong to the provided [RetrievableId]. + * + * @param retrievableId The [RetrievableId] to search for. + * @return [Sequence] of [Descriptor] of type [D] + */ + override fun getForRetrievable(retrievableId: RetrievableId): Sequence { + try { + this.connection.jdbc.prepareStatement("SELECT * FROM $tableName WHERE $RETRIEVABLE_ID_COLUMN_NAME = ?").use { stmt -> + stmt.setObject(1, retrievableId) + val result = stmt.executeQuery() + return sequence { + result.use { + while (result.next()) { + yield(this@AbstractDescriptorReader.rowToDescriptor(result)) + } + } + } + } + } catch (e: Exception) { + LOGGER.error(e) { "Failed to fetch descriptor for retrievable $retrievableId from '$tableName' due to SQL error." } + return emptySequence() + } + } + + /** + * Checks whether a [Descriptor] of type [D] with the provided [UUID] exists. + * + * @param descriptorId The [DescriptorId], i.e., the [UUID] of the [Descriptor] to check for. + * @return True if descriptor exsits, false otherwise + */ + override fun exists(descriptorId: DescriptorId): Boolean { + try { + this.connection.jdbc.prepareStatement("SELECT count(*) FROM $tableName WHERE $DESCRIPTOR_ID_COLUMN_NAME = ?").use { stmt -> + stmt.setObject(1, descriptorId) + stmt.executeQuery().use { res -> + res.next() + return res.getLong(1) > 0L + } + } + } catch (e: Exception) { + LOGGER.error(e) { "Failed to check for descriptor $descriptorId in '$tableName' due to SQL error." } + return false + } + } + + /** + * Returns all [Descriptor]s of type [D] as a [Sequence]. + * + * @return [Sequence] of all [Descriptor]s. + */ + override fun getAll(): Sequence { + try { + this.connection.jdbc.prepareStatement("SELECT * FROM $tableName").use { stmt -> + val result = stmt.executeQuery() + return sequence { + result.use { + while (result.next()) { + yield(this@AbstractDescriptorReader.rowToDescriptor(result)) + } + } + } + } + } catch (e: Exception) { + LOGGER.error(e) { "Failed to fetch descriptors from '$tableName' due to SQL error." } + return emptySequence() + } + } + + /** + * Returns a [Sequence] of all [Descriptor]s whose [DescriptorId] is contained in the provided [Iterable]. + * + * @param descriptorIds A [Iterable] of [DescriptorId]s to return. + * @return [Sequence] of [Descriptor] of type [D] + */ + override fun getAll(descriptorIds: Iterable): Sequence { + try { + this.connection.jdbc.prepareStatement("SELECT * FROM $tableName WHERE $DESCRIPTOR_ID_COLUMN_NAME = ANY (?)").use { stmt -> + val values = descriptorIds.map { it }.toTypedArray() + stmt.setArray(1, this.connection.jdbc.createArrayOf("uuid", values)) + val result = stmt.executeQuery() + return sequence { + result.use { + while (result.next()) { + yield(this@AbstractDescriptorReader.rowToDescriptor(result)) + } + } + } + } + } catch (e: Exception) { + LOGGER.error(e) { "Failed to fetch descriptors from '$tableName' due to SQL error." } + return emptySequence() + } + } + + /** + * Returns a [Sequence] of all [Descriptor] whose [RetrievableId] is contained in the provided [Iterable]. + * + * @param retrievableIds A [Iterable] of [RetrievableId]s to return [Descriptor]s for + * @return [Sequence] of [Descriptor] of type [D] + */ + override fun getAllForRetrievable(retrievableIds: Iterable): Sequence { + try { + this.connection.jdbc.prepareStatement("SELECT * FROM $tableName WHERE $RETRIEVABLE_ID_COLUMN_NAME = ANY (?)").use { stmt -> + val values = retrievableIds.map { it }.toTypedArray() + stmt.setArray(1, this.connection.jdbc.createArrayOf("uuid", values)) + val result = stmt.executeQuery() + return sequence { + result.use { + while (result.next()) { + yield(this@AbstractDescriptorReader.rowToDescriptor(result)) + } + } + } + } + } catch (e: Exception) { + LOGGER.error(e) { "Failed to fetch descriptors from '$tableName' due to SQL error." } + return emptySequence() + } + } + + /** + * Returns the number of [Descriptor]s contained in the entity managed by this [AbstractDescriptorReader] + * + * @return [Sequence] of all [Descriptor]s. + */ + override fun count(): Long { + try { + this.connection.jdbc.prepareStatement("SELECT COUNT(*) FROM $tableName;").use { stmt -> + stmt.executeQuery().use { result -> + result.next() + return result.getLong(1) + } + } + } catch (e: Exception) { + LOGGER.error(e) { "Failed to count retrievable due to SQL error." } + return 0L + } + } + + /** + * Returns a [Sequence] of all [Retrieved]s that match the given [Query]. + * + * Implicitly, this methods executes a [query] and then JOINS the result with the [Retrieved]s. + * + * @param query The [Query] that should be executed. + * @return [Sequence] of [Retrieved]. + */ + override fun queryAndJoin(query: Query): Sequence { + val descriptors = query(query).toList() + if (descriptors.isEmpty()) return emptySequence() + + /* Fetch retrievable ids. */ + val retrievables = this.connection.getRetrievableReader().getAll(descriptors.mapNotNull { it.retrievableId }.toSet()).map { it.id to it }.toMap() + return descriptors.asSequence().mapNotNull { descriptor -> + val retrievable = retrievables[descriptor.retrievableId] + if (retrievable != null) { + retrievable.addDescriptor(descriptor) + retrievable as Retrieved + } else { + null + } + } + } + + /** + * Converts a [ResultSet] to a [Descriptor] of type [D]. + * + * @param result The [ResultSet] to convert. + * @return [Descriptor] of type [D] + */ + abstract fun rowToDescriptor(result: ResultSet): D +} \ No newline at end of file diff --git a/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/PgDescriptorInitializer.kt b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/PgDescriptorInitializer.kt new file mode 100644 index 00000000..827a39a7 --- /dev/null +++ b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/PgDescriptorInitializer.kt @@ -0,0 +1,158 @@ +package org.vitrivr.engine.database.pgvector.descriptor + +import org.vitrivr.engine.core.config.schema.IndexType +import org.vitrivr.engine.core.database.Initializer.Companion.DISTANCE_PARAMETER_NAME +import org.vitrivr.engine.core.database.Initializer.Companion.INDEX_TYPE_PARAMETER_NAME +import org.vitrivr.engine.core.database.descriptor.DescriptorInitializer +import org.vitrivr.engine.core.model.descriptor.Descriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.query.basics.Distance +import org.vitrivr.engine.core.model.types.Type +import org.vitrivr.engine.database.pgvector.* +import java.sql.SQLException + +/** + * An abstract implementation of a [DescriptorInitializer] for PostgreSQL with pgVector. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +open class PgDescriptorInitializer(final override val field: Schema.Field<*, D>, protected val connection: PgVectorConnection): DescriptorInitializer { + + companion object { + /** Set of scalar index structures supported by PostgreSQL. */ + private val INDEXES_SCALAR = setOf("btree", "brin", "hash") + + /** Set of NNS index structures supported by PostgreSQL. */ + private val INDEXES_NNS = setOf("hnsw", "ivfflat") + } + + /** The name of the table backing this [PgDescriptorInitializer]. */ + protected val tableName: String = "${DESCRIPTOR_ENTITY_PREFIX}_${this.field.fieldName}" + + /** The [Descriptor] prototype for this [PgDescriptorWriter]. */ + protected val prototype = this.field.analyser.prototype(this.field) + + /** + * Initializes the PostgreSQL table entity backing this [PgDescriptorInitializer]. + */ + override fun initialize() { + val statement = StringBuilder("CREATE TABLE IF NOT EXISTS \"${tableName}\" (") + statement.append("$DESCRIPTOR_ID_COLUMN_NAME uuid NOT NULL, ") + statement.append("$RETRIEVABLE_ID_COLUMN_NAME uuid NOT NULL, ") + + /* Add columns for each field in the struct. */ + for (field in this.prototype.layout()) { + when (field.type) { + Type.String -> statement.append("\"${field.name}\" varchar(255), ") + Type.Text -> statement.append("\"${field.name}\" text, ") + Type.Boolean -> statement.append("\"${field.name}\" boolean, ") + Type.Byte -> statement.append("\"${field.name}\" smallint, ") + Type.Short -> statement.append("\"${field.name}\" smallint, ") + Type.Int -> statement.append("\"${field.name}\" integer, ") + Type.Long -> statement.append("\"${field.name}\" bigint, ") + Type.Float -> statement.append("\"${field.name}\" real, ") + Type.Double -> statement.append("\"${field.name}\" double precision, ") + Type.Datetime -> statement.append("\"${field.name}\" datetime, ") + is Type.BooleanVector -> statement.append("\"${field.name}\" bit(${field.type.dimensions}), ") + is Type.DoubleVector -> statement.append("\"${field.name}\" vector(${field.type.dimensions}), ") + is Type.FloatVector -> statement.append("\"${field.name}\" vector(${field.type.dimensions}), ") + is Type.IntVector -> statement.append("\"${field.name}\" vector(${field.type.dimensions}), ") + is Type.LongVector -> statement.append("\"${field.name}\" vector(${field.type.dimensions}), ") + } + } + + /* Finalize statement*/ + statement.append("PRIMARY KEY ($DESCRIPTOR_ID_COLUMN_NAME), ") + statement.append("FOREIGN KEY ($RETRIEVABLE_ID_COLUMN_NAME) REFERENCES $RETRIEVABLE_ENTITY_NAME($RETRIEVABLE_ID_COLUMN_NAME) ON DELETE CASCADE);") + + /* Create entity. */ + try { + this.connection.jdbc.prepareStatement(/* sql = postgres */ statement.toString()).use { + it.execute() + } + } catch (e: SQLException) { + LOGGER.error(e) { "Failed to initialize entity '$tableName' due to exception." } + throw e + } + + /* Create indexes (optional). */ + for (index in this.field.indexes) { + try { + val indexStatement = when (index.type) { + IndexType.SCALAR -> { + val type = index.parameters[INDEX_TYPE_PARAMETER_NAME]?.lowercase() ?: "btree" + require(type in INDEXES_SCALAR) { "Index type '$type' is not supported by PostgreSQL." } + "CREATE INDEX ON $tableName USING $type(${index.attributes.joinToString(",")});" + } + IndexType.FULLTEXT -> "CREATE INDEX ON $tableName USING gin(${index.attributes.joinToString(",") { "to_tsvector('${index.parameters["language"] ?: "english"}', $it)" }});" + IndexType.NNS -> { + require(index.attributes.size == 1) { "NNS index can only be created on a single attribute." } + val type = index.parameters[INDEX_TYPE_PARAMETER_NAME]?.lowercase() ?: "hnsw" + val distance = index.parameters[DISTANCE_PARAMETER_NAME]?.let { Distance.valueOf(it.uppercase()) } ?: Distance.EUCLIDEAN + require(type in INDEXES_NNS) { "Index type '$type' is not supported by PostgreSQL." } + "CREATE INDEX ON $tableName USING $type(${index.attributes.first()} ${distance.toIndexName()});" + } + } + this.connection.jdbc.prepareStatement(/* sql = postgres */ indexStatement).use { it.execute() } + } catch (e: SQLException) { + LOGGER.error(e) { "Failed to create index ${index.type} for entity '$tableName' due to exception." } + throw e + } + } + } + + /** + * De-initializes the PostgreSQL table entity backing this [PgDescriptorInitializer]. + */ + override fun deinitialize() { + try { + /* Create 'retrievable' entity and index. */ + this.connection.jdbc.prepareStatement(/* sql = postgres */ "DROP TABLE IF EXISTS \"${tableName}\" CASCADE;").use { + it.execute() + } + } catch (e: SQLException) { + LOGGER.error(e) { "Failed to de-initialize entity '$tableName' due to exception." } + } + } + + /** + * Checks if the schema for this [PgDescriptorInitializer] has been properly initialized. + * + * @return True if entity has been initialized, false otherwise. + */ + override fun isInitialized(): Boolean { + try { + this.connection.jdbc.prepareStatement(/* sql = postgres */ "SELECT count(*) FROM $tableName").use { + it.execute() + } + } catch (e: SQLException) { + return false + } + return true + } + + /** + * Truncates the table backing this [PgDescriptorInitializer]. + */ + override fun truncate() { + try { + this.connection.jdbc.prepareStatement(/* sql = postgres */ "TRUNCATE $tableName").use { + it.execute() + } + } catch (e: SQLException) { + LOGGER.error(e) { "Failed to truncate entities due to exception." } + } + } + + /** + * Closes the [PgDescriptorInitializer]. + */ + private fun Distance.toIndexName() = when (this) { + Distance.MANHATTAN -> "vector_l1_ops" + Distance.EUCLIDEAN -> "sparsevec_l2_ops" + Distance.COSINE -> "vector_cosine_ops" + Distance.HAMMING -> "bit_hamming_ops" + Distance.JACCARD -> "bit_jaccard_ops" + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/PgDescriptorWriter.kt b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/PgDescriptorWriter.kt new file mode 100644 index 00000000..b2300d00 --- /dev/null +++ b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/PgDescriptorWriter.kt @@ -0,0 +1,179 @@ +package org.vitrivr.engine.database.pgvector.descriptor + +import org.vitrivr.engine.core.database.descriptor.DescriptorWriter +import org.vitrivr.engine.core.model.descriptor.Descriptor +import org.vitrivr.engine.core.model.descriptor.struct.StructDescriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.database.pgvector.* +import java.sql.* + +/** + * An abstract implementation of a [DescriptorWriter] for PostgreSQL with pgVector. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +open class PgDescriptorWriter(final override val field: Schema.Field<*, D>, override val connection: PgVectorConnection): DescriptorWriter { + /** The name of the table backing this [PgDescriptorInitializer]. */ + protected val tableName: String = "${DESCRIPTOR_ENTITY_PREFIX}_${this.field.fieldName}" + + /** The [Descriptor] prototype for this [PgDescriptorWriter]. */ + protected val prototype = this.field.analyser.prototype(this.field) + + /** + * Adds (writes) a single [StructDescriptor] using this [StructDescriptorWriter]. + * + * @param item The [StructDescriptor] to write. + * @return True on success, false otherwise. + */ + override fun add(item: D): Boolean { + try { + this.prepareInsertStatement().use { stmt -> + stmt.setObject(1, item.id) + stmt.setObject(2, item.retrievableId) + var i = 3 + for (attribute in item.layout()) { + val value = item.values()[attribute.name] + if (value != null) { + stmt.setValue(i++, value) + } else { + stmt.setNull(i++, attribute.type.toSql()) + } + } + return stmt.executeUpdate() == 1 + } + } catch (e: SQLException) { + LOGGER.error(e) { "Failed to INSERT descriptor ${item.id} into '$tableName' due to SQL error." } + return false + } + } + + /** + * Adds (writes) a batch of [Descriptor] of type [D] using this [StructDescriptorWriter]. + * + * @param items A [Iterable] of [Descriptor]s to write. + * @return True on success, false otherwise. + */ + override fun addAll(items: Iterable): Boolean { + try { + this.prepareInsertStatement().use { stmt -> + for (item in items) { + stmt.setObject(1, item.id) + stmt.setObject(2, item.retrievableId) + var i = 3 + for (attribute in item.layout()) { + val value = item.values()[attribute.name] + if (value != null) { + stmt.setValue(i++, value) + } else { + stmt.setNull(i++, attribute.type.toSql()) + } + } + stmt.addBatch() + } + return stmt.executeBatch().all { it == 1 } + } + } catch (e: SQLException) { + LOGGER.error(e) { "Failed to INSERT descriptors into '$tableName' due to SQL error." } + return false + } + } + + /** + * Updates a specific [Descriptor] of type [D] using this [StructDescriptorWriter]. + * + * @param item A [Descriptor]s to update. + * @return True on success, false otherwise. + */ + override fun update(item: D): Boolean { + try { + this.prepareUpdateStatement().use { stmt -> + stmt.setObject(1, item.retrievableId) + var i = 2 + for (attribute in item.layout()) { + val value = item.values()[attribute.name] + if (value != null) { + stmt.setValue(i++, value) + } else { + stmt.setNull(i++, attribute.type.toSql()) + } + } + stmt.setObject(i, item.id) + return stmt.execute() + } + } catch (e: SQLException) { + LOGGER.error(e) { "Failed to UPDATE descriptors in '$tableName' due to SQL error." } + return false + } + } + + /** + * Deletes (writes) a [Descriptor] of type [D] using this [PgDescriptorWriter]. + * + * @param item A [Descriptor]s to delete. + * @return True on success, false otherwise. + */ + override fun delete(item: D): Boolean { + try { + this.connection.jdbc.prepareStatement("DELETE FROM $tableName WHERE $DESCRIPTOR_ID_COLUMN_NAME = ?;").use { stmt -> + stmt.setObject(1, item.id) + return stmt.executeUpdate() == 1 + } + } catch (e: SQLException) { + LOGGER.error(e) { "Failed to delete descriptor ${item.id} due to SQL error." } + return false + } + } + + /** + * Deletes (writes) [Descriptor]s of type [D] using this [PgDescriptorWriter]. + * + * @param items A [Iterable] of [Descriptor]s to delete. + * @return True on success, false otherwise. + */ + override fun deleteAll(items: Iterable): Boolean { + try { + this.connection.jdbc.prepareStatement("DELETE FROM $tableName WHERE $DESCRIPTOR_ID_COLUMN_NAME = ANY (?);").use { stmt -> + val values = items.map { it.id }.toTypedArray() + stmt.setArray(1, this.connection.jdbc.createArrayOf("uuid", values)) + return stmt.executeUpdate() > 0 + } + } catch (e: SQLException) { + LOGGER.error(e) { "Failed to delete descriptors due to SQL error." } + return false + } + } + + + /** + * Prepares an INSERT statement for this [StructDescriptorWriter]. + * + * @return [PreparedStatement] + */ + protected fun prepareUpdateStatement(): PreparedStatement { + val statement = StringBuilder("UPDATE $tableName SET $RETRIEVABLE_ID_COLUMN_NAME = ?") + for (field in this.prototype.layout()) { + statement.append(", \"${field.name}\" = ?") + } + statement.append("WHERE $DESCRIPTOR_ID_COLUMN_NAME = ?;") + return this.connection.jdbc.prepareStatement(statement.toString()) + } + + /** + * Prepares an INSERT statement for this [StructDescriptorWriter]. + * + * @return [PreparedStatement] + */ + protected fun prepareInsertStatement(): PreparedStatement { + val statement = StringBuilder("INSERT INTO $tableName ($DESCRIPTOR_ID_COLUMN_NAME, $RETRIEVABLE_ID_COLUMN_NAME") + for (field in this.prototype.layout()) { + statement.append(", \"${field.name}\"") + } + statement.append(") VALUES (?, ?") + for (field in this.field.analyser.prototype(this.field).layout()) { + statement.append(", ?") + } + statement.append(");") + return this.connection.jdbc.prepareStatement(statement.toString()) + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/model/PgBitVector.kt b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/model/PgBitVector.kt new file mode 100644 index 00000000..6a608820 --- /dev/null +++ b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/model/PgBitVector.kt @@ -0,0 +1,162 @@ +package org.vitrivr.engine.database.pgvector.descriptor.model + +import org.postgresql.util.ByteConverter +import org.postgresql.util.PGBinaryObject +import org.postgresql.util.PGobject +import org.vitrivr.engine.core.model.types.Value +import java.io.Serializable +import java.sql.SQLException + + +/** + * The [PgBitVector] class represents a bit vector in PostgreSQL. + * + * @link https://github.com/pgvector/pgvector-java/blob/master/src/main/java/com/pgvector/PGbit.java + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class PgBitVector() : PGobject(), PGBinaryObject, Serializable, Cloneable { + + /** The [ByteArray] backing this [PgBitVector]. */ + private var vec: ByteArray? = null + + /** The length of this [PgBitVector]. */ + private var length = 0 + + /** + * Constructor + */ + init { + this.type = "bit" + } + + /** + * Constructor + * + * @param v boolean array + */ + constructor(v: BooleanArray) : this() { + val vec = ByteArray((v.size + 7) / 8) + for (i in v.indices) { + vec[i / 8] = (vec[i / 8].toInt() or ((if (v[i]) 1 else 0) shl (7 - (i % 8)))).toByte() + } + + /* Update local state. */ + this.vec = vec + this.length = v.size + } + + /** + * Constructor + * + * @param s text representation of a bit string + * @throws SQLException exception + */ + constructor(s: String?) : this() { + setValue(s) + } + + /** + * Sets the value from a text representation of a bit string + */ + @Throws(SQLException::class) + override fun setValue(s: String?) { + if (s == null) { + this.vec = null + this.length = 0 + } else { + val data = ByteArray((s.length + 7) / 8) + for (i in s.indices) { + data[i / 8] = (data[i / 8].toInt() or ((if (s[i] != '0') 1 else 0) shl (7 - (i % 8)))).toByte() + } + /* Update local state. */ + this.vec = data + this.length = s.length + } + } + + /** + * Returns the text representation of a bit string + */ + override fun getValue(): String? { + val data = this.vec ?: return null + val sb = StringBuilder(length) + for (i in 0 until length) { + sb.append(if (((data[i / 8].toInt() shr (7 - (i % 8))) and 1) == 1) '1' else '0') + } + return sb.toString() + } + + /** + * Returns the number of bytes for the binary representation + */ + override fun lengthInBytes(): Int = this.vec?.size?.plus(4) ?: 0 + + /** + * Sets the value from a binary representation of a bit string + */ + @Throws(SQLException::class) + override fun setByteValue(value: ByteArray, offset: Int) { + val length = ByteConverter.int4(value, offset) + val data = ByteArray((length + 7) / 8) + for (i in data.indices) { + data[i] = value[offset + 4 + i] + } + + /* Update local state. */ + this.vec = data + this.length = length + } + + /** + * Writes the binary representation of a bit string + */ + override fun toBytes(bytes: ByteArray, offset: Int) { + val data = this.vec ?: return + ByteConverter.int4(bytes, offset, length) + for (i in data.indices) { + bytes[offset + 4 + i] = data[i] + } + } + + /** + * Clones this [PgBitVector]. + * + * @return Copy of this [PgBitVector]. + */ + override fun clone(): PgBitVector { + val clone = PgBitVector() + clone.vec = this.vec?.copyOf() + clone.length = this.length + return clone + } + + /** + * Returns the length + * + * @return an array + */ + fun length(): Int = this.length + + /** + * Returns a byte array + * + * @return an array + */ + fun toByteArray(): ByteArray? = this.vec + + /** + * Returns a [Value.BooleanVector] representation of this [PgBitVector]. + * + * @return [Value.BooleanVector] + */ + fun toBooleanVector(): Value.BooleanVector? { + if (this.vec == null) return null + val bits = BooleanArray(this.length) + for (i in 0 until this.length) { + bits[i] = ((this.vec!![i / 8].toInt() shr (7 - (i % 8))) and 1) == 1 + } + return Value.BooleanVector(bits) + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/model/PgVector.kt b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/model/PgVector.kt new file mode 100644 index 00000000..dc458c5a --- /dev/null +++ b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/model/PgVector.kt @@ -0,0 +1,172 @@ +package org.vitrivr.engine.database.pgvector.descriptor.model + +import org.postgresql.util.ByteConverter +import org.postgresql.util.PGBinaryObject +import org.postgresql.util.PGobject +import org.vitrivr.engine.core.model.types.Value +import java.io.Serializable +import java.sql.SQLException +import java.util.* + +/** + * The [PgVector] class represents a vector in PostgreSQL. + * + * @link https://github.com/pgvector/pgvector-java/blob/master/src/main/java/com/pgvector/PGvector.java + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class PgVector(private var vec: FloatArray? = null) : PGobject(), PGBinaryObject, Serializable, Cloneable { + + init { + this.type = "vector" + } + + /** + * Constructor + * + * @param number + * @param v list of numbers + */ + constructor(vector: List) : this(FloatArray(vector.size) { + when (val v = vector[it]) { + is Number -> v.toFloat() + is Value.Float-> v.value + is Value.Double-> v.value.toFloat() + is Value.Int-> v.value.toFloat() + is Value.Long-> v.value.toFloat() + else -> throw IllegalArgumentException("Could not convert $v to float.") + } + }) + + /** + * Constructor + * + * @param vector [DoubleArray] + */ + constructor(vector: DoubleArray) : this(FloatArray(vector.size) { + vector[it].toFloat() + }) + + /** + * Constructor + * + * @param vector [IntArray] + */ + constructor(vector: IntArray) : this(FloatArray(vector.size) { + vector[it].toFloat() + }) + + /** + * Constructor + * + * @param vector [LongArray] + */ + constructor(vector: LongArray) : this(FloatArray(vector.size) { + vector[it].toFloat() + }) + + /** + * Constructor + * + * @param s text representation of a vector + * @throws SQLException exception + */ + constructor(s: String?) : this() { + setValue(s) + } + + /** + * Sets the value from a text representation of a vector + */ + @Throws(SQLException::class) + override fun setValue(s: String?) { + if (s == null) { + this.vec = null + } else { + val sp = s.substring(1, s.length - 1).split(",".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray() + this.vec = FloatArray(sp.size) + for (i in sp.indices) { + vec!![i] = sp[i].toFloat() + } + } + } + + /** + * Returns the text representation of a vector + */ + override fun getValue(): String? = this.vec?.contentToString()?.replace(" ", "") + + /** + * Returns the number of bytes for the binary representation + */ + override fun lengthInBytes(): Int = if (vec == null) 0 else 4 + vec!!.size * 4 + + /** + * Sets the value from a binary representation of a vector + */ + @Throws(SQLException::class) + override fun setByteValue(value: ByteArray, offset: Int) { + val dim: Short = ByteConverter.int2(value, offset) + val unused: Short = ByteConverter.int2(value, offset + 2) + if (unused != 0.toShort()) { + throw SQLException("expected unused to be 0") + } + + this.vec = FloatArray(dim.toInt()) { + ByteConverter.float4(value, offset + 4 + it * 4) + } + } + + /** + * Writes the binary representation of a vector + */ + override fun toBytes(bytes: ByteArray, offset: Int) { + if (vec == null) { + return + } + + // server will error on overflow due to unconsumed buffer + // could set to Short.MAX_VALUE for friendlier error message + ByteConverter.int2(bytes, offset, vec!!.size) + ByteConverter.int2(bytes, offset + 2, 0) + for (i in this.vec!!.indices) { + ByteConverter.float4(bytes, offset + 4 + i * 4, vec!![i]) + } + } + + /** + * Clones this [PgVector]. + * + * @return Copy of this [PgVector]. + */ + override fun clone(): PgVector = PgVector(this.vec?.copyOf()) + + /** + * Returns an [Value.FloatVector] representation of this [PgVector]. + * + * @return [Value.FloatVector] + */ + fun toFloatVector(): Value.FloatVector? = this.vec?.let { Value.FloatVector(it) } + + /** + * Returns an [Value.DoubleVector] representation of this [PgVector]. + * + * @return [Value.DoubleVector] + */ + fun toDoubleVector(): Value.DoubleVector? = this.vec?.let { vec -> Value.DoubleVector(DoubleArray(vec.size) { i -> vec[i].toDouble()}) } + + /** + * Returns an [Value.IntVector] representation of this [PgVector]. + * + * @return [Value.IntVector] + */ + fun toIntVector(): Value.IntVector? = this.vec?.let { vec -> Value.IntVector(IntArray(vec.size) { i -> vec[i].toInt()}) } + + /** + * Returns an [Value.LongVector] representation of this [PgVector]. + * + * @return [Value.LongVector] + */ + fun toLongVector(): Value.LongVector? = this.vec?.let { vec -> Value.LongVector(LongArray(vec.size) { i -> vec[i].toLong()}) } +} \ No newline at end of file diff --git a/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/scalar/ScalarDescriptorProvider.kt b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/scalar/ScalarDescriptorProvider.kt new file mode 100644 index 00000000..8fb2ff59 --- /dev/null +++ b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/scalar/ScalarDescriptorProvider.kt @@ -0,0 +1,22 @@ +package org.vitrivr.engine.database.pgvector.descriptor.scalar + +import org.vitrivr.engine.core.database.Connection +import org.vitrivr.engine.core.database.descriptor.DescriptorProvider +import org.vitrivr.engine.core.model.descriptor.scalar.ScalarDescriptor +import org.vitrivr.engine.core.model.descriptor.struct.StructDescriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.database.pgvector.PgVectorConnection +import org.vitrivr.engine.database.pgvector.descriptor.PgDescriptorInitializer +import org.vitrivr.engine.database.pgvector.descriptor.PgDescriptorWriter + +/** + * A [DescriptorProvider] for [StructDescriptor]s. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +object ScalarDescriptorProvider: DescriptorProvider> { + override fun newInitializer(connection: Connection, field: Schema.Field<*, ScalarDescriptor<*>>) = PgDescriptorInitializer(field, connection as PgVectorConnection) + override fun newReader(connection: Connection, field: Schema.Field<*, ScalarDescriptor<*>>) = ScalarDescriptorReader(field, connection as PgVectorConnection) + override fun newWriter(connection: Connection, field: Schema.Field<*, ScalarDescriptor<*>>) = PgDescriptorWriter(field, connection as PgVectorConnection) +} \ No newline at end of file diff --git a/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/scalar/ScalarDescriptorReader.kt b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/scalar/ScalarDescriptorReader.kt new file mode 100644 index 00000000..82d3f39b --- /dev/null +++ b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/scalar/ScalarDescriptorReader.kt @@ -0,0 +1,96 @@ +package org.vitrivr.engine.database.pgvector.descriptor.scalar + +import org.vitrivr.engine.core.model.descriptor.scalar.* +import org.vitrivr.engine.core.model.descriptor.scalar.ScalarDescriptor.Companion.VALUE_ATTRIBUTE_NAME +import org.vitrivr.engine.core.model.descriptor.struct.StructDescriptor +import org.vitrivr.engine.core.model.descriptor.vector.VectorDescriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.query.Query +import org.vitrivr.engine.core.model.query.bool.SimpleBooleanQuery +import org.vitrivr.engine.core.model.query.fulltext.SimpleFulltextQuery +import org.vitrivr.engine.core.model.retrievable.Retrieved +import org.vitrivr.engine.core.model.types.Value +import org.vitrivr.engine.database.pgvector.* +import org.vitrivr.engine.database.pgvector.descriptor.AbstractDescriptorReader +import java.sql.ResultSet +import java.util.* + +/** + * A [AbstractDescriptorReader] for [ScalarDescriptor]s. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class ScalarDescriptorReader(field: Schema.Field<*, ScalarDescriptor<*>>, connection: PgVectorConnection) : AbstractDescriptorReader>(field, connection) { + + /** + * Executes the provided [Query] and returns a [Sequence] of [Retrieved]s that match it. + * + * @param query The [Query] to execute. + * @return [Sequence] of [StructDescriptor]s that match the query. + */ + override fun query(query: Query): Sequence> = when (query) { + is SimpleFulltextQuery -> queryFulltext(query) + is SimpleBooleanQuery<*> -> queryBoolean(query) + else -> throw IllegalArgumentException("Query of type ${query::class} is not supported by ScalarDescriptorReader.") + } + + /** + * Converts the provided [ResultSet] to a [VectorDescriptor]. + * + * @param result The [ResultSet] to convert. + * @return The resulting [VectorDescriptor]. + */ + override fun rowToDescriptor(result: ResultSet): ScalarDescriptor<*> { + val descriptorId = result.getObject(DESCRIPTOR_ID_COLUMN_NAME, UUID::class.java) + val retrievableId = result.getObject(RETRIEVABLE_ID_COLUMN_NAME, UUID::class.java) + return when (this.prototype) { + is BooleanDescriptor -> BooleanDescriptor(descriptorId, retrievableId, Value.Boolean(result.getBoolean(VALUE_ATTRIBUTE_NAME))) + is IntDescriptor -> IntDescriptor(descriptorId, retrievableId, Value.Int(result.getInt(VALUE_ATTRIBUTE_NAME))) + is LongDescriptor -> LongDescriptor(descriptorId, retrievableId, Value.Long(result.getLong(VALUE_ATTRIBUTE_NAME))) + is FloatDescriptor -> FloatDescriptor(descriptorId, retrievableId, Value.Float(result.getFloat(VALUE_ATTRIBUTE_NAME))) + is DoubleDescriptor -> DoubleDescriptor(descriptorId, retrievableId, Value.Double(result.getDouble(VALUE_ATTRIBUTE_NAME))) + is StringDescriptor -> StringDescriptor(descriptorId, retrievableId, Value.String(result.getString(VALUE_ATTRIBUTE_NAME))) + } + } + + /** + * Executes a [SimpleFulltextQuery] and returns a [Sequence] of [ScalarDescriptor]s. + * + * @param query The [SimpleFulltextQuery] to execute. + * @return [Sequence] of [ScalarDescriptor]s. + */ + private fun queryFulltext(query: SimpleFulltextQuery): Sequence> { + val statement = "SELECT * FROM \"$tableName\" WHERE $VALUE_ATTRIBUTE_NAME @@ plainto_tsquery(?)" + return sequence { + this@ScalarDescriptorReader.connection.jdbc.prepareStatement(statement).use { stmt -> + stmt.setString(1, query.value.value) + stmt.executeQuery().use { result -> + while (result.next()) { + yield(rowToDescriptor(result)) + } + } + } + } + } + + /** + * Executes a [SimpleBooleanQuery] and returns a [Sequence] of [ScalarDescriptor]s. + * + * @param query The [SimpleBooleanQuery] to execute. + * @return [Sequence] of [ScalarDescriptor]s. + */ + private fun queryBoolean(query: SimpleBooleanQuery<*>): Sequence> { + val statement = "SELECT * FROM \"$tableName\" WHERE $VALUE_ATTRIBUTE_NAME ${query.comparison.toSql()} ?" + return sequence { + this@ScalarDescriptorReader.connection.jdbc.prepareStatement(statement).use { stmt -> + stmt.setValue(1, query.value) + stmt.executeQuery().use { result -> + while (result.next()) { + yield(rowToDescriptor(result)) + } + } + } + } + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/struct/StructDescriptorProvider.kt b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/struct/StructDescriptorProvider.kt new file mode 100644 index 00000000..edbccb8c --- /dev/null +++ b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/struct/StructDescriptorProvider.kt @@ -0,0 +1,22 @@ +package org.vitrivr.engine.database.pgvector.descriptor.struct + +import org.vitrivr.engine.core.database.Connection +import org.vitrivr.engine.core.database.descriptor.DescriptorProvider +import org.vitrivr.engine.core.model.descriptor.struct.LabelDescriptor +import org.vitrivr.engine.core.model.descriptor.struct.StructDescriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.database.pgvector.PgVectorConnection +import org.vitrivr.engine.database.pgvector.descriptor.PgDescriptorInitializer +import org.vitrivr.engine.database.pgvector.descriptor.PgDescriptorWriter + +/** + * A [DescriptorProvider] for [LabelDescriptor]s. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +object StructDescriptorProvider : DescriptorProvider { + override fun newInitializer(connection: Connection, field: Schema.Field<*, StructDescriptor>) = PgDescriptorInitializer(field, connection as PgVectorConnection) + override fun newReader(connection: Connection, field: Schema.Field<*, StructDescriptor>) = StructDescriptorReader(field, connection as PgVectorConnection) + override fun newWriter(connection: Connection, field: Schema.Field<*, StructDescriptor>) = PgDescriptorWriter(field, connection as PgVectorConnection) +} \ No newline at end of file diff --git a/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/struct/StructDescriptorReader.kt b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/struct/StructDescriptorReader.kt new file mode 100644 index 00000000..d9870e62 --- /dev/null +++ b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/struct/StructDescriptorReader.kt @@ -0,0 +1,121 @@ +package org.vitrivr.engine.database.pgvector.descriptor.struct + +import org.vitrivr.engine.core.model.descriptor.AttributeName +import org.vitrivr.engine.core.model.descriptor.struct.StructDescriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.query.Query +import org.vitrivr.engine.core.model.query.bool.SimpleBooleanQuery +import org.vitrivr.engine.core.model.query.fulltext.SimpleFulltextQuery +import org.vitrivr.engine.core.model.retrievable.Retrieved +import org.vitrivr.engine.core.model.types.Type +import org.vitrivr.engine.core.model.types.Value +import org.vitrivr.engine.database.pgvector.* +import org.vitrivr.engine.database.pgvector.descriptor.AbstractDescriptorReader +import org.vitrivr.engine.database.pgvector.descriptor.model.PgBitVector +import org.vitrivr.engine.database.pgvector.descriptor.model.PgVector +import java.sql.ResultSet +import java.util.* +import kotlin.reflect.full.primaryConstructor + +/** + * An [AbstractDescriptorReader] for [StructDescriptor]s. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class StructDescriptorReader(field: Schema.Field<*, StructDescriptor>, connection: PgVectorConnection) : AbstractDescriptorReader(field, connection) { + /** + * Executes the provided [Query] and returns a [Sequence] of [Retrieved]s that match it. + * + * @param query The [Query] to execute. + * @return [Sequence] of [StructDescriptor]s that match the query. + */ + override fun query(query: Query): Sequence = when (query) { + is SimpleFulltextQuery -> queryFulltext(query) + is SimpleBooleanQuery<*> -> queryBoolean(query) + else -> throw IllegalArgumentException("Query of typ ${query::class} is not supported by StructDescriptorReader.") + } + + /** + * Converts the provided [ResultSet] to a [StructDescriptor]. + * + * @param result The [ResultSet] to convert. + * @return The resulting [StructDescriptor]. + */ + override fun rowToDescriptor(result: ResultSet): StructDescriptor { + val constructor = this.field.analyser.descriptorClass.primaryConstructor ?: throw IllegalStateException("Provided type ${this.field.analyser.descriptorClass} does not have a primary constructor.") + val values = TreeMap?>() + val parameters: MutableList = mutableListOf( + result.getObject(DESCRIPTOR_ID_COLUMN_NAME, UUID::class.java) ?: throw IllegalArgumentException("The provided tuple is missing the required field '${DESCRIPTOR_ID_COLUMN_NAME}'."), + result.getObject(RETRIEVABLE_ID_COLUMN_NAME, UUID::class.java) ?: throw IllegalArgumentException("The provided tuple is missing the required field '${RETRIEVABLE_ID_COLUMN_NAME}'."), + values, + this.field + ) + + /* Append dynamic parameters of struct. */ + for (field in this.prototype.layout()) { + values[field.name] = when(field.type) { + Type.String -> result.getString(field.name)?.let { Value.String(it) } + Type.Text -> result.getString(field.name)?.let { Value.Text(it) } + Type.Boolean -> result.getBoolean(field.name).let { Value.Boolean(it) } + Type.Byte -> result.getByte(field.name).let { Value.Byte(it) } + Type.Short -> result.getShort(field.name).let { Value.Short(it) } + Type.Int -> result.getInt(field.name).let { Value.Int(it) } + Type.Long -> result.getLong(field.name).let { Value.Long(it) } + Type.Float -> result.getFloat(field.name).let { Value.Float(it) } + Type.Double -> result.getDouble(field.name).let { Value.Double(it) } + Type.Datetime -> result.getDate(field.name).toInstant().let { Value.DateTime(Date(it.toEpochMilli())) } + is Type.BooleanVector -> result.getObject(field.name, PgBitVector::class.java).toBooleanVector() + is Type.IntVector -> result.getObject(field.name, PgVector::class.java)?.toIntVector() + is Type.LongVector -> result.getObject(field.name, PgVector::class.java)?.toLongVector() + is Type.FloatVector -> result.getObject(field.name, PgVector::class.java)?.toFloatVector() + is Type.DoubleVector -> result.getObject(field.name, PgVector::class.java)?.toDoubleVector() + } as Value<*>? + } + + /* Call constructor. */ + return constructor.call(*parameters.toTypedArray()) + } + + /** + * Executes a [SimpleFulltextQuery] and returns a [Sequence] of [StructDescriptor]s. + * + * @param query The [SimpleFulltextQuery] to execute. + * @return [Sequence] of [StructDescriptor]s. + */ + private fun queryFulltext(query: SimpleFulltextQuery): Sequence { + require(query.attributeName != null) { "Query attribute must not be null for a fulltext query on a struct descriptor." } + val statement = "SELECT * FROM \"$tableName\" WHERE ${query.attributeName} @@ plainto_tsquery(?)" + return sequence { + this@StructDescriptorReader.connection.jdbc.prepareStatement(statement).use { stmt -> + stmt.setString(1, query.value.value) + stmt.executeQuery().use { result -> + while (result.next()) { + yield(rowToDescriptor(result)) + } + } + } + } + } + + /** + * Executes a [SimpleBooleanQuery] and returns a [Sequence] of [StructDescriptor]s. + * + * @param query The [SimpleBooleanQuery] to execute. + * @return [Sequence] of [StructDescriptor]s. + */ + private fun queryBoolean(query: SimpleBooleanQuery<*>): Sequence { + require(query.attributeName != null) { "Query attribute must not be null for a fulltext query on a struct descriptor." } + val statement = "SELECT * FROM \"$tableName\" WHERE ${query.attributeName} ${query.comparison.toSql()} ?" + return sequence { + this@StructDescriptorReader.connection.jdbc.prepareStatement(statement).use { stmt -> + stmt.setValue(1, query.value) + stmt.executeQuery().use { result -> + while (result.next()) { + yield(rowToDescriptor(result)) + } + } + } + } + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/vector/VectorDescriptorProvider.kt b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/vector/VectorDescriptorProvider.kt new file mode 100644 index 00000000..57b7a00a --- /dev/null +++ b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/vector/VectorDescriptorProvider.kt @@ -0,0 +1,21 @@ +package org.vitrivr.engine.database.pgvector.descriptor.vector + +import org.vitrivr.engine.core.database.Connection +import org.vitrivr.engine.core.database.descriptor.DescriptorProvider +import org.vitrivr.engine.core.model.descriptor.vector.VectorDescriptor +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.database.pgvector.PgVectorConnection +import org.vitrivr.engine.database.pgvector.descriptor.PgDescriptorInitializer +import org.vitrivr.engine.database.pgvector.descriptor.PgDescriptorWriter + +/** + * A [DescriptorProvider] for [VectorDescriptor]s. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +object VectorDescriptorProvider: DescriptorProvider> { + override fun newInitializer(connection: Connection, field: Schema.Field<*, VectorDescriptor<*>>) = PgDescriptorInitializer(field, connection as PgVectorConnection) + override fun newReader(connection: Connection, field: Schema.Field<*, VectorDescriptor<*>>) = VectorDescriptorReader(field, connection as PgVectorConnection) + override fun newWriter(connection: Connection, field: Schema.Field<*, VectorDescriptor<*>>) = PgDescriptorWriter(field, connection as PgVectorConnection) +} \ No newline at end of file diff --git a/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/vector/VectorDescriptorReader.kt b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/vector/VectorDescriptorReader.kt new file mode 100644 index 00000000..6d69d35b --- /dev/null +++ b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/descriptor/vector/VectorDescriptorReader.kt @@ -0,0 +1,145 @@ +package org.vitrivr.engine.database.pgvector.descriptor.vector + +import org.vitrivr.engine.core.database.descriptor.DescriptorReader +import org.vitrivr.engine.core.model.descriptor.vector.* +import org.vitrivr.engine.core.model.descriptor.vector.VectorDescriptor.Companion.VECTOR_ATTRIBUTE_NAME +import org.vitrivr.engine.core.model.metamodel.Schema +import org.vitrivr.engine.core.model.query.Query +import org.vitrivr.engine.core.model.query.proximity.ProximityQuery +import org.vitrivr.engine.core.model.retrievable.Retrieved +import org.vitrivr.engine.core.model.retrievable.attributes.DistanceAttribute +import org.vitrivr.engine.database.pgvector.* +import org.vitrivr.engine.database.pgvector.descriptor.AbstractDescriptorReader +import org.vitrivr.engine.database.pgvector.descriptor.model.PgBitVector +import org.vitrivr.engine.database.pgvector.descriptor.model.PgVector +import java.sql.ResultSet +import java.util.* + +/** + * An abstract implementation of a [DescriptorReader] for Cottontail DB. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class VectorDescriptorReader(field: Schema.Field<*, VectorDescriptor<*>>, connection: PgVectorConnection) : AbstractDescriptorReader>(field, connection) { + /** + * Executes the provided [Query] and returns a [Sequence] of [Retrieved]s that match it. + * + * @param query The [Query] to execute. + */ + override fun query(query: Query): Sequence> = when (query) { + is ProximityQuery<*> -> queryProximity(query) + else -> throw UnsupportedOperationException("Query of typ ${query::class} is not supported by VectorDescriptorReader.") + } + + /** + * Returns a [Sequence] of all [Retrieved]s that match the given [Query]. + * + * Implicitly, this methods executes a [query] and then JOINS the result with the [Retrieved]s. + * + * @param query The [Query] that should be executed. + * @return [Sequence] of [Retrieved]. + */ + override fun queryAndJoin(query: Query): Sequence = when (query) { + is ProximityQuery<*> -> queryAndJoinProximity(query) + else -> super.queryAndJoin(query) + } + + /** + * Converts a [ResultSet] to a [VectorDescriptor]. + * + * @param result [ResultSet] to convert. + * @return [VectorDescriptor] + */ + override fun rowToDescriptor(result: ResultSet): VectorDescriptor<*> { + val descriptorId = result.getObject(DESCRIPTOR_ID_COLUMN_NAME, UUID::class.java) + val retrievableId = result.getObject(RETRIEVABLE_ID_COLUMN_NAME, UUID::class.java) + return when (this.prototype) { + is FloatVectorDescriptor -> FloatVectorDescriptor( + descriptorId, + retrievableId, + result.getObject(VECTOR_ATTRIBUTE_NAME, PgVector::class.java)?.toFloatVector() ?: throw IllegalArgumentException("The provided vector value is missing the required field '$VECTOR_ATTRIBUTE_NAME'.") + ) + + is DoubleVectorDescriptor -> DoubleVectorDescriptor( + descriptorId, + retrievableId, + result.getObject(VECTOR_ATTRIBUTE_NAME, PgVector::class.java)?.toDoubleVector() ?: throw IllegalArgumentException("The provided vector value is missing the required field '$VECTOR_ATTRIBUTE_NAME'.") + ) + + is IntVectorDescriptor -> IntVectorDescriptor( + descriptorId, + retrievableId, + result.getObject(VECTOR_ATTRIBUTE_NAME, PgVector::class.java)?.toIntVector() ?: throw IllegalArgumentException("The provided vector value is missing the required field '$VECTOR_ATTRIBUTE_NAME'.") + ) + + is LongVectorDescriptor -> LongVectorDescriptor( + descriptorId, + retrievableId, + result.getObject(VECTOR_ATTRIBUTE_NAME, PgVector::class.java)?.toLongVector() ?: throw IllegalArgumentException("The provided vector value is missing the required field '$VECTOR_ATTRIBUTE_NAME'.") + ) + + is BooleanVectorDescriptor -> BooleanVectorDescriptor( + descriptorId, + retrievableId, + result.getObject(VECTOR_ATTRIBUTE_NAME, PgBitVector::class.java)?.toBooleanVector() ?: throw IllegalArgumentException("The provided vector value is missing the required field '$VECTOR_ATTRIBUTE_NAME'.") + ) + + else -> throw IllegalArgumentException("Unsupported descriptor type ${this.prototype::class}.") + } + } + + /** + * Executes a [ProximityQuery] and returns a [Sequence] of [VectorDescriptor]s. + * + * @param query The [ProximityQuery] to execute. + * @return [Sequence] of [VectorDescriptor]s. + */ + private fun queryProximity(query: ProximityQuery<*>): Sequence> = sequence { + val statement = + "SELECT $DESCRIPTOR_ID_COLUMN_NAME, $RETRIEVABLE_ID_COLUMN_NAME, $VECTOR_ATTRIBUTE_NAME, $VECTOR_ATTRIBUTE_NAME ${query.distance.toSql()} ? AS $DISTANCE_COLUMN_NAME FROM \"${tableName}\" ORDER BY $DISTANCE_COLUMN_NAME ${query.order} LIMIT ${query.k}" + this@VectorDescriptorReader.connection.jdbc.prepareStatement(statement).use { stmt -> + stmt.setValue(1, query.value) + stmt.executeQuery().use { result -> + while (result.next()) { + yield(rowToDescriptor(result)) + } + } + } + } + + /** + * Executes a [ProximityQuery] and returns a [Sequence] of [VectorDescriptor]s. + * + * @param query The [ProximityQuery] to execute. + * @return [Sequence] of [VectorDescriptor]s. + */ + private fun queryAndJoinProximity(query: ProximityQuery<*>): Sequence { + val descriptors = mutableListOf, Float>>() + val statement = + "SELECT $DESCRIPTOR_ID_COLUMN_NAME, $RETRIEVABLE_ID_COLUMN_NAME, $VECTOR_ATTRIBUTE_NAME, $VECTOR_ATTRIBUTE_NAME ${query.distance.toSql()} ? AS $DISTANCE_COLUMN_NAME FROM \"${tableName}\" ORDER BY $DISTANCE_COLUMN_NAME ${query.order} LIMIT ${query.k}" + this@VectorDescriptorReader.connection.jdbc.prepareStatement(statement).use { stmt -> + stmt.setValue(1, query.value) + stmt.executeQuery().use { result -> + while (result.next()) { + descriptors.add(this@VectorDescriptorReader.rowToDescriptor(result) to result.getFloat(DISTANCE_COLUMN_NAME)) + } + } + + /* Fetch retrievable ids. */ + val retrievables = this.connection.getRetrievableReader().getAll(descriptors.mapNotNull { it.first.retrievableId }.toSet()).map { it.id to it }.toMap() + return descriptors.asSequence().mapNotNull { (descriptor, distance) -> + val retrievable = retrievables[descriptor.retrievableId] + if (retrievable != null) { + if (query.fetchVector) { + retrievable.addDescriptor(descriptor) + } + retrievable.addAttribute(DistanceAttribute(distance)) + retrievable as Retrieved + } else { + null + } + } + } + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/retrievable/RetrievableInitializer.kt b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/retrievable/RetrievableInitializer.kt new file mode 100644 index 00000000..afd8408f --- /dev/null +++ b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/retrievable/RetrievableInitializer.kt @@ -0,0 +1,89 @@ +package org.vitrivr.engine.database.pgvector.retrievable + +import org.vitrivr.engine.core.database.retrievable.RetrievableInitializer +import org.vitrivr.engine.core.model.retrievable.Retrievable +import org.vitrivr.engine.database.pgvector.* +import java.sql.SQLException + +/** + * A [RetrievableInitializer] implementation for PostgreSQL with pgVector. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +internal class RetrievableInitializer(private val connection: PgVectorConnection): RetrievableInitializer { + /** + * Initializes the data structures backing this [RetrievableInitializer]. + */ + override fun initialize() { + try { + /* Create 'retrievable' entity and index. */ + this.connection.jdbc.prepareStatement(/* sql = postgres */ "CREATE TABLE IF NOT EXISTS \"${RETRIEVABLE_ENTITY_NAME}\" ($RETRIEVABLE_ID_COLUMN_NAME uuid NOT NULL, type VARCHAR(100), PRIMARY KEY ($RETRIEVABLE_ID_COLUMN_NAME));").use { + it.execute() + } + + /* Create 'relationship' entity. */ + this.connection.jdbc.prepareStatement(/* sql = postgres */ "CREATE TABLE IF NOT EXISTS \"${RELATIONSHIP_ENTITY_NAME}\" ($OBJECT_ID_COLUMN_NAME uuid NOT NULL, $PREDICATE_COLUMN_NAME VARCHAR(100) NOT NULL, $SUBJECT_ID_COLUMN_NAME uuid NOT NULL, PRIMARY KEY ($OBJECT_ID_COLUMN_NAME, $PREDICATE_COLUMN_NAME, $SUBJECT_ID_COLUMN_NAME), FOREIGN KEY($OBJECT_ID_COLUMN_NAME) REFERENCES $RETRIEVABLE_ENTITY_NAME($RETRIEVABLE_ID_COLUMN_NAME) ON DELETE CASCADE, FOREIGN KEY($SUBJECT_ID_COLUMN_NAME) REFERENCES $RETRIEVABLE_ENTITY_NAME($RETRIEVABLE_ID_COLUMN_NAME) ON DELETE CASCADE);") + .use { + it.execute() + } + } catch (e: SQLException) { + LOGGER.error(e) { "Failed to initialize entity due to exception." } + } + } + + /** + * De-initializes the data structures backing this [RetrievableInitializer]. + */ + override fun deinitialize() { + try { + /* Create 'retrievable' entity and index. */ + this.connection.jdbc.prepareStatement(/* sql = postgres */ "DROP TABLE IF EXISTS \"${RETRIEVABLE_ENTITY_NAME}\" CASCADE;").use { + it.execute() + } + + /* Create 'relationship' entity. */ + this.connection.jdbc.prepareStatement(/* sql = postgres */ "DROP TABLE IF EXISTS \"${RELATIONSHIP_ENTITY_NAME}\" CASCADE;").use { + it.execute() + } + } catch (e: SQLException) { + LOGGER.error(e) { "Failed to initialize entity due to exception." } + } + } + + /** + * Checks if the schema for this [RetrievableInitializer] has been properly initialized. + * + * @return True if entity has been initialized, false otherwise. + */ + override fun isInitialized(): Boolean { + try { + this.connection.jdbc.prepareStatement(/* sql = postgres */ "SELECT count(*) FROM \"${RETRIEVABLE_ENTITY_NAME}\";").use { + it.execute() + } + } catch (e: SQLException) { + return false + } + try { + this.connection.jdbc.prepareStatement(/* sql = postgres */ "SELECT count(*) FROM \"$RELATIONSHIP_ENTITY_NAME\";").use { + it.execute() + } + } catch (e: SQLException) { + return false + } + return true + } + + /** + * Truncates the entity that is used to store [Retrievable]s and Relationships in PostgreSQL with pgVector. + */ + override fun truncate() { + try { + this.connection.jdbc.prepareStatement(/* sql = postgres */ "TRUNCATE \"${RETRIEVABLE_ENTITY_NAME}\", \"${RELATIONSHIP_ENTITY_NAME}\"").use { + it.execute() + } + } catch (e: SQLException) { + LOGGER.error(e) { "Failed to truncate entities due to exception." } + } + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/retrievable/RetrievableReader.kt b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/retrievable/RetrievableReader.kt new file mode 100644 index 00000000..2d479018 --- /dev/null +++ b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/retrievable/RetrievableReader.kt @@ -0,0 +1,132 @@ +package org.vitrivr.engine.database.pgvector.retrievable + +import org.vitrivr.engine.core.database.retrievable.RetrievableReader +import org.vitrivr.engine.core.model.retrievable.Retrievable +import org.vitrivr.engine.core.model.retrievable.RetrievableId +import org.vitrivr.engine.core.model.retrievable.Retrieved +import org.vitrivr.engine.database.pgvector.* +import java.util.* + +/** + * A [RetrievableReader] implementation for PostgreSQL with pgVector. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class RetrievableReader(override val connection: PgVectorConnection): RetrievableReader { + /** + * Returns the [Retrievable]s that matches the provided [RetrievableId] + */ + override fun get(id: RetrievableId): Retrievable? { + try { + this.connection.jdbc.prepareStatement("SELECT * FROM $RETRIEVABLE_ENTITY_NAME WHERE $RETRIEVABLE_ID_COLUMN_NAME = ?").use { stmt -> + stmt.setObject(1, id) + stmt.executeQuery().use { res -> + if (res.next() ) { + return Retrieved(res.getObject(RETRIEVABLE_ID_COLUMN_NAME, UUID::class.java), res.getString(RETRIEVABLE_TYPE_COLUMN_NAME), false) + } else { + return null + } + } + } + } catch (e: Exception) { + LOGGER.error(e) { "Failed to check for retrievable $id due to SQL error." } + return null + } + } + + /** + * Checks whether a [Retrievable] with the provided [RetrievableId] exists. + * + * @param id The [RetrievableId], i.e., the [UUID] of the [Retrievable] to check for. + * @return True if descriptor exists, false otherwise + */ + override fun exists(id: RetrievableId): Boolean { + try { + this.connection.jdbc.prepareStatement("SELECT count(*) FROM $RETRIEVABLE_ENTITY_NAME WHERE $RETRIEVABLE_ID_COLUMN_NAME = ?").use { stmt -> + stmt.setObject(1, id) + stmt.executeQuery().use { res -> + res.next() + return res.getLong(1) > 0L + } + } + } catch (e: Exception) { + LOGGER.error(e) { "Failed to check for retrievable $id due to SQL error." } + return false + } + } + + /** + * Returns all [Retrievable]s that match any of the provided [RetrievableId] + * + * @param ids A [Iterable] of [RetrievableId]s to return. + * @return A [Sequence] of all [Retrievable]. + */ + override fun getAll(ids: Iterable): Sequence { + try { + val statement = this.connection.jdbc.prepareStatement("SELECT * FROM $RETRIEVABLE_ENTITY_NAME WHERE $RETRIEVABLE_ID_COLUMN_NAME = ANY (?)") + val values = ids.map { it }.toTypedArray() + statement.setArray(1, this.connection.jdbc.createArrayOf("uuid", values)) + return sequence { + val result = statement.executeQuery() + while (result.next()) { + yield(Retrieved(result.getObject(RETRIEVABLE_ID_COLUMN_NAME, UUID::class.java), result.getString(RETRIEVABLE_TYPE_COLUMN_NAME), false)) + } + result.close() + statement.close() + } + } catch (e: Exception) { + LOGGER.error(e) { "Failed to check for retrievables due to SQL error." } + return emptySequence() + } + } + + /** + * Returns all [Retrievable]s stored by the database. + * + * @return A [Sequence] of all [Retrievable]s in the database. + */ + override fun getAll(): Sequence { + try { + this.connection.jdbc.prepareStatement("SELECT * FROM $RETRIEVABLE_ENTITY_NAME").use { stmt -> + val result = stmt.executeQuery() + return sequence { + while (result.next()) { + yield(Retrieved(result.getObject(RETRIEVABLE_ID_COLUMN_NAME, UUID::class.java), result.getString(RETRIEVABLE_TYPE_COLUMN_NAME), false)) + } + result.close() + } + } + } catch (e: Exception) { + LOGGER.error(e) { "Failed to check for retrievables due to SQL error." } + return emptySequence() + } + } + + override fun getConnections( + subjectIds: Collection, + predicates: Collection, + objectIds: Collection + ): Sequence> { + TODO("Not yet implemented") + } + + /** + * Counts the number of [Retrievable] stored by the database. + * + * @return The number of [Retrievable]s. + */ + override fun count(): Long { + try { + this.connection.jdbc.prepareStatement("SELECT COUNT(*) FROM $RETRIEVABLE_ENTITY_NAME;").use { stmt -> + stmt.executeQuery().use { result -> + result.next() + return result.getLong(1) + } + } + } catch (e: Exception) { + LOGGER.error(e) { "Failed to count retrievable due to SQL error." } + return 0L + } + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/retrievable/RetrievableWriter.kt b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/retrievable/RetrievableWriter.kt new file mode 100644 index 00000000..5de14b54 --- /dev/null +++ b/vitrivr-engine-module-pgvector/src/main/kotlin/org/vitrivr/engine/database/pgvector/retrievable/RetrievableWriter.kt @@ -0,0 +1,181 @@ +package org.vitrivr.engine.database.pgvector.retrievable + +import org.vitrivr.engine.core.database.retrievable.RetrievableWriter +import org.vitrivr.engine.core.model.relationship.Relationship +import org.vitrivr.engine.core.model.retrievable.Retrievable +import org.vitrivr.engine.database.pgvector.* +import java.sql.SQLException + +/** + * A [RetrievableWriter] implementation for PostgreSQL with pgVector. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +internal class RetrievableWriter(override val connection: PgVectorConnection): RetrievableWriter { + /** + * Adds a new [Retrievable] to the database using this [RetrievableWriter] instance. + * + * @param item [Retrievable] to add. + */ + override fun add(item: Retrievable): Boolean { + try { + this.connection.jdbc.prepareStatement("INSERT INTO $RETRIEVABLE_ENTITY_NAME ($RETRIEVABLE_ID_COLUMN_NAME, $RETRIEVABLE_TYPE_COLUMN_NAME) VALUES (?, ?);").use { stmt -> + stmt.setObject(1, item.id) + stmt.setString(2, item.type) + return stmt.executeUpdate() == 1 + } + } catch (e: SQLException) { + LOGGER.error(e) { "Failed to persist retrievable ${item.id} due to SQL error." } + return false + } + } + + /** + * Adds new [Retrievable]s to the database using this [RetrievableWriter] instance. + * + * @param items An [Iterable] of [Retrievable]s to add. + */ + override fun addAll(items: Iterable): Boolean { + try { + this.connection.jdbc.prepareStatement("INSERT INTO $RETRIEVABLE_ENTITY_NAME ($RETRIEVABLE_ID_COLUMN_NAME, $RETRIEVABLE_TYPE_COLUMN_NAME) VALUES (?, ?);").use { stmt -> + for (item in items) { + stmt.setObject(1, item.id) + stmt.setString(2, item.type) + stmt.addBatch() + } + return stmt.executeBatch().all { it == 1 } + } + } catch (e: SQLException) { + LOGGER.error(e) { "Failed to persist retrievables due to SQL error." } + return false + } + } + + /** + * Updates a specific [Retrievable] using this [RetrievableWriter]. + * + * @param item A [Retrievable]s to update. + * @return True on success, false otherwise. + */ + override fun update(item: Retrievable): Boolean { + try { + this.connection.jdbc.prepareStatement("UPDATE $RETRIEVABLE_ENTITY_NAME SET $RETRIEVABLE_TYPE_COLUMN_NAME = ? WHERE $RETRIEVABLE_ID_COLUMN_NAME = ?").use { stmt -> + stmt.setString(1, item.type) + stmt.setObject(2, item.id) + return stmt.executeUpdate() == 1 + } + } catch (e: SQLException) { + LOGGER.error(e) { "Failed to update retrievable ${item.id} due to SQL error." } + return false + } + } + + /** + * Deletes (writes) a [Retrievable] using this [RetrievableWriter]. + * + * @param item A [Retrievable]s to delete. + * @return True on success, false otherwise. + */ + override fun delete(item: Retrievable): Boolean { + try { + this.connection.jdbc.prepareStatement("DELETE FROM $RETRIEVABLE_ENTITY_NAME WHERE $RETRIEVABLE_ID_COLUMN_NAME = ?;").use { stmt -> + stmt.setObject(1, item.id) + return stmt.executeUpdate() > 0 + } + } catch (e: SQLException) { + LOGGER.error(e) { "Failed to delete retrievable ${item.id} due to SQL error." } + return false + } + } + + /** + * Deletes (writes) a iterable of [Retrievable]s using this [RetrievableWriter]. + * + * @param items A [Iterable] of [Retrievable]s to delete. + * @return True on success, false otherwise. + */ + override fun deleteAll(items: Iterable): Boolean { + try { + this.connection.jdbc.prepareStatement("DELETE FROM $RETRIEVABLE_ENTITY_NAME WHERE $RETRIEVABLE_ID_COLUMN_NAME = ANY (?);").use { stmt -> + val values = items.map { it.id }.toTypedArray() + stmt.setArray(1, this.connection.jdbc.createArrayOf("uuid", values)) + return stmt.executeUpdate() > 0 + } + } catch (e: SQLException) { + LOGGER.error(e) { "Failed to delete retrievable due to SQL error." } + return false + } + } + + /** + * Persists a [Relationship]. + * + * @param relationship [Relationship] to persist + * @return True on success, false otherwise. + */ + override fun connect(relationship: Relationship): Boolean { + try { + this.connection.jdbc.prepareStatement("INSERT INTO $RELATIONSHIP_ENTITY_NAME ($OBJECT_ID_COLUMN_NAME,$PREDICATE_COLUMN_NAME,$SUBJECT_ID_COLUMN_NAME) VALUES (?,?,?)").use { stmt -> + stmt.setObject(1, relationship.objectId) + stmt.setString(2, relationship.predicate) + stmt.setObject(3, relationship.subjectId) + return stmt.execute() + } + } catch (e: SQLException) { + LOGGER.error(e) { "Failed to insert relationship ${relationship.objectId} -(${relationship.predicate}-> ${relationship.subjectId} due to SQL error." } + return false + } + } + + /** + * Persists an [Iterable] of [Relationship]s. + * + * @param relationships An [Iterable] of [Relationship]s to persist. + * @return True on success, false otherwise. + */ + override fun connectAll(relationships: Iterable): Boolean { + try { + this.connection.jdbc.prepareStatement("INSERT INTO $RELATIONSHIP_ENTITY_NAME ($OBJECT_ID_COLUMN_NAME,$PREDICATE_COLUMN_NAME,$SUBJECT_ID_COLUMN_NAME) VALUES (?,?,?)").use { stmt -> + for (relationship in relationships) { + stmt.setObject(1, relationship.objectId) + stmt.setString(2, relationship.predicate) + stmt.setObject(3, relationship.subjectId) + stmt.addBatch() + } + return stmt.executeBatch().all { it == 1 } + } + } catch (e: SQLException) { + LOGGER.error(e) { "Failed to insert relationships due to SQL error." } + return false + } + } + + /** + * Severs the specified connection between two [Retrievable]s. + * + * @param relationship [Relationship] to delete + * @return True on success, false otherwise. + */ + override fun disconnect(relationship: Relationship): Boolean { + try { + this.connection.jdbc.prepareStatement("DELETE FROM $RELATIONSHIP_ENTITY_NAME WHERE $OBJECT_ID_COLUMN_NAME = ? AND $PREDICATE_COLUMN_NAME = ? AND $SUBJECT_ID_COLUMN_NAME = ?").use { stmt -> + stmt.setObject(1, relationship.objectId) + stmt.setString(2, relationship.predicate) + stmt.setObject(3, relationship.subjectId) + return stmt.execute() + } + } catch (e: SQLException) { + LOGGER.error(e) { "Failed to delete relationship ${relationship.objectId} -(${relationship.predicate}-> ${relationship.subjectId} due to SQL error." } + return false + } + } + + /** + * Deletes all [Relationship]s + * + * @param relationships An [Iterable] of [Relationship] to delete. + * @return True on success, false otherwise. + */ + override fun disconnectAll(relationships: Iterable): Boolean = relationships.map { this.disconnect(it) }.all { it } +} \ No newline at end of file diff --git a/vitrivr-engine-module-pgvector/src/main/resources/META-INF/services/org.vitrivr.engine.core.database.ConnectionProvider b/vitrivr-engine-module-pgvector/src/main/resources/META-INF/services/org.vitrivr.engine.core.database.ConnectionProvider new file mode 100644 index 00000000..497e745a --- /dev/null +++ b/vitrivr-engine-module-pgvector/src/main/resources/META-INF/services/org.vitrivr.engine.core.database.ConnectionProvider @@ -0,0 +1 @@ +org.vitrivr.engine.database.pgvector.PgVectorConnectionProvider \ No newline at end of file diff --git a/vitrivr-engine-module-pgvector/src/test/kotlin/org/vitrivr/engine/database/pgvector/descriptor/DescriptorInitializerTest.kt b/vitrivr-engine-module-pgvector/src/test/kotlin/org/vitrivr/engine/database/pgvector/descriptor/DescriptorInitializerTest.kt new file mode 100644 index 00000000..92a9d757 --- /dev/null +++ b/vitrivr-engine-module-pgvector/src/test/kotlin/org/vitrivr/engine/database/pgvector/descriptor/DescriptorInitializerTest.kt @@ -0,0 +1,11 @@ +package org.vitrivr.engine.database.pgvector.descriptor + +import org.vitrivr.engine.core.database.descriptor.AbstractDescriptorInitializerTest + +/** + * An [AbstractDescriptorInitializerTest] implementation for PostgreSQL with pgVector. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class DescriptorInitializerTest : AbstractDescriptorInitializerTest("test-schema-postgres.json") \ No newline at end of file diff --git a/vitrivr-engine-module-pgvector/src/test/kotlin/org/vitrivr/engine/database/pgvector/descriptor/struct/FileMetadataDescriptorReaderTest.kt b/vitrivr-engine-module-pgvector/src/test/kotlin/org/vitrivr/engine/database/pgvector/descriptor/struct/FileMetadataDescriptorReaderTest.kt new file mode 100644 index 00000000..20155d50 --- /dev/null +++ b/vitrivr-engine-module-pgvector/src/test/kotlin/org/vitrivr/engine/database/pgvector/descriptor/struct/FileMetadataDescriptorReaderTest.kt @@ -0,0 +1,11 @@ +package org.vitrivr.engine.database.pgvector.descriptor.struct + +import org.vitrivr.engine.core.database.descriptor.struct.AbstractFileMetadataDescriptorReaderTest + +/** + * An [AbstractFileMetadataDescriptorReaderTest] implementation for PostgreSQL with pgVector. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class FileMetadataDescriptorReaderTest : AbstractFileMetadataDescriptorReaderTest("test-schema-postgres.json") \ No newline at end of file diff --git a/vitrivr-engine-module-pgvector/src/test/kotlin/org/vitrivr/engine/database/pgvector/descriptor/vector/FloatVectorDescriptorReaderTest.kt b/vitrivr-engine-module-pgvector/src/test/kotlin/org/vitrivr/engine/database/pgvector/descriptor/vector/FloatVectorDescriptorReaderTest.kt new file mode 100644 index 00000000..6586ee9e --- /dev/null +++ b/vitrivr-engine-module-pgvector/src/test/kotlin/org/vitrivr/engine/database/pgvector/descriptor/vector/FloatVectorDescriptorReaderTest.kt @@ -0,0 +1,11 @@ +package org.vitrivr.engine.database.pgvector.descriptor.vector + +import org.vitrivr.engine.core.database.descriptor.vector.AbstractFloatVectorDescriptorReaderTest + +/** + * An [AbstractFloatVectorDescriptorReaderTest] implementation for PostgreSQL with pgVector. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class FloatVectorDescriptorReaderTest : AbstractFloatVectorDescriptorReaderTest("test-schema-postgres.json") \ No newline at end of file diff --git a/vitrivr-engine-module-pgvector/src/test/kotlin/org/vitrivr/engine/database/pgvector/retrievable/RetrievableInitializerTest.kt b/vitrivr-engine-module-pgvector/src/test/kotlin/org/vitrivr/engine/database/pgvector/retrievable/RetrievableInitializerTest.kt new file mode 100644 index 00000000..35be2e7e --- /dev/null +++ b/vitrivr-engine-module-pgvector/src/test/kotlin/org/vitrivr/engine/database/pgvector/retrievable/RetrievableInitializerTest.kt @@ -0,0 +1,53 @@ +package org.vitrivr.engine.database.pgvector.retrievable + +import org.junit.jupiter.api.Assertions +import org.vitrivr.engine.core.database.retrievable.AbstractRetrievableInitializerTest +import org.vitrivr.engine.database.pgvector.PgVectorConnection +import org.vitrivr.engine.database.pgvector.RELATIONSHIP_ENTITY_NAME +import org.vitrivr.engine.database.pgvector.RETRIEVABLE_ENTITY_NAME + +/** + * An [AbstractRetrievableInitializerTest] for the [PgVectorConnection]. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class RetrievableInitializerTest : AbstractRetrievableInitializerTest("test-schema-postgres.json") { + + /** + * + */ + override fun testInitializeEntities() { + super.testInitializeEntities() + Assertions.assertTrue(checkTablesExist()) + } + + /** + * Checks if tables initialized actually exists. + * + * @return True if tables exist, false otherwise. + */ + private fun checkTablesExist(): Boolean { + /* Check for existence of tables. */ + (this.testConnection as PgVectorConnection).jdbc.prepareStatement("SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_schema = ? AND table_name = ?)").use { statement -> + /* Check existence of retrievable table. */ + statement.setString(1, SCHEMA_NAME) + statement.setString(2, RETRIEVABLE_ENTITY_NAME) + statement.executeQuery().use { result -> + if (!result.next() || !result.getBoolean(1)) { + return false + } + } + + /* Check existence of relationship table. */ + statement.setString(1, SCHEMA_NAME) + statement.setString(2, RELATIONSHIP_ENTITY_NAME) + statement.executeQuery().use { result -> + if (!result.next() || !result.getBoolean(1)) { + return false + } + } + } + return true + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-pgvector/src/test/kotlin/org/vitrivr/engine/database/pgvector/retrievable/RetrievableWriterTest.kt b/vitrivr-engine-module-pgvector/src/test/kotlin/org/vitrivr/engine/database/pgvector/retrievable/RetrievableWriterTest.kt new file mode 100644 index 00000000..c6326f32 --- /dev/null +++ b/vitrivr-engine-module-pgvector/src/test/kotlin/org/vitrivr/engine/database/pgvector/retrievable/RetrievableWriterTest.kt @@ -0,0 +1,13 @@ +package org.vitrivr.engine.database.pgvector.retrievable + +import org.vitrivr.engine.core.database.retrievable.AbstractRetrievableWriterTest + +/** + * An [AbstractRetrievableWriterTest] for the [PgVectorConnection]. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class RetrievableWriterTest : AbstractRetrievableWriterTest("test-schema-postgres.json") { + +} \ No newline at end of file diff --git a/vitrivr-engine-module-pgvector/src/test/resources/test-schema-postgres.json b/vitrivr-engine-module-pgvector/src/test/resources/test-schema-postgres.json new file mode 100644 index 00000000..7e69198d --- /dev/null +++ b/vitrivr-engine-module-pgvector/src/test/resources/test-schema-postgres.json @@ -0,0 +1,45 @@ +{ + "name": "vitrivr-test", + "connection": { + "database": "PgVectorConnectionProvider", + "parameters": { + "host": "127.0.0.1", + "port": "5432", + "username": "postgres", + "password": "vitrivr" + } + }, + "fields": [ + { + "name": "averagecolor", + "factory": "AverageColor" + }, + { + "name": "file", + "factory": "FileSourceMetadata", + "indexes": [ + { + "attributes": [ + "path" + ], + "type": "FULLTEXT" + }, + { + "attributes": [ + "size" + ], + "type": "SCALAR" + } + ] + }, + { + "name": "time", + "factory": "TemporalMetadata" + }, + { + "name": "video", + "factory": "VideoSourceMetadata" + } + ], + "exporters": [] +} diff --git a/vitrivr-engine-query/src/main/kotlin/org/vitrivr/engine/query/model/api/input/InputData.kt b/vitrivr-engine-query/src/main/kotlin/org/vitrivr/engine/query/model/api/input/InputData.kt index d149b197..342f9625 100644 --- a/vitrivr-engine-query/src/main/kotlin/org/vitrivr/engine/query/model/api/input/InputData.kt +++ b/vitrivr-engine-query/src/main/kotlin/org/vitrivr/engine/query/model/api/input/InputData.kt @@ -104,7 +104,7 @@ data class RetrievableIdInputData(val id: String, override val comparison: Strin * Cannot be converted to a [ContentElement] */ @Serializable -data class BooleanInputData(val value: Boolean, override val comparison: String? = "=="): InputData(){ +data class BooleanInputData(val data: Boolean, override val comparison: String? = "=="): InputData(){ override val type = InputType.BOOLEAN override fun toContent(): ContentElement<*> { throw UnsupportedOperationException("Cannot derive content from BooleanInputData") @@ -116,7 +116,7 @@ data class BooleanInputData(val value: Boolean, override val comparison: String? * Cannot be converted to a [ContentElement] */ @Serializable -data class NumericInputData(val value: Double, override val comparison: String? = "==") : InputData(){ +data class NumericInputData(val data: Double, override val comparison: String? = "==") : InputData(){ override val type = InputType.NUMERIC override fun toContent(): ContentElement<*> { throw UnsupportedOperationException("Cannot derive content from NumericInputData") @@ -128,7 +128,7 @@ data class NumericInputData(val value: Double, override val comparison: String? * Cannot be converted to a [ContentElement] */ @Serializable -data class DateInputData(val value: String, override val comparison: String? = "==") : InputData() { +data class DateInputData(val data: String, override val comparison: String? = "==") : InputData() { override val type = InputType.DATE override fun toContent(): ContentElement<*> {throw UnsupportedOperationException("Cannot derive content from DateInputData")} @@ -137,6 +137,6 @@ data class DateInputData(val value: String, override val comparison: String? = " */ fun parseDate():Date{ val formatter = SimpleDateFormat("YYYY-mm-dd", Locale.ENGLISH) - return formatter.parse(value) + return formatter.parse(data) } } diff --git a/vitrivr-engine-query/src/main/kotlin/org/vitrivr/engine/query/operators/transform/lookup/FieldLookup.kt b/vitrivr-engine-query/src/main/kotlin/org/vitrivr/engine/query/operators/transform/lookup/FieldLookup.kt index 06b7d1d4..9b14d214 100644 --- a/vitrivr-engine-query/src/main/kotlin/org/vitrivr/engine/query/operators/transform/lookup/FieldLookup.kt +++ b/vitrivr-engine-query/src/main/kotlin/org/vitrivr/engine/query/operators/transform/lookup/FieldLookup.kt @@ -35,7 +35,7 @@ class FieldLookup( val descriptors = if (ids.isEmpty()) { emptyMap() } else { - this@FieldLookup.reader.getAllFor(ids).associateBy { it.retrievableId!! } + this@FieldLookup.reader.getAllForRetrievable(ids).associateBy { it.retrievableId!! } } /* Emit retrievable with added attribute. */ diff --git a/vitrivr-engine-query/src/main/kotlin/org/vitrivr/engine/query/operators/transform/lookup/FieldLookupFactory.kt b/vitrivr-engine-query/src/main/kotlin/org/vitrivr/engine/query/operators/transform/lookup/FieldLookupFactory.kt index c2bef129..0e1cd0c0 100644 --- a/vitrivr-engine-query/src/main/kotlin/org/vitrivr/engine/query/operators/transform/lookup/FieldLookupFactory.kt +++ b/vitrivr-engine-query/src/main/kotlin/org/vitrivr/engine/query/operators/transform/lookup/FieldLookupFactory.kt @@ -15,9 +15,9 @@ class FieldLookupFactory() : TransformerFactory { val providedKeys = context[name, "keys"] val keys = if(providedKeys?.isNotBlank() == true){ if(providedKeys.length == 1 && providedKeys == "*") { - schemaField.analyser.prototype(schemaField).schema().map { it.name } + schemaField.analyser.prototype(schemaField).layout().map { it.name } }else{ - providedKeys.split(",").map { s -> s.trim() } ?: emptyList() + providedKeys.split(",").map { s -> s.trim() } } }else{ emptyList() diff --git a/vitrivr-engine-query/src/main/kotlin/org/vitrivr/engine/query/operators/transform/lookup/ObjectFieldLookup.kt b/vitrivr-engine-query/src/main/kotlin/org/vitrivr/engine/query/operators/transform/lookup/ObjectFieldLookup.kt index c8aff560..cda01633 100644 --- a/vitrivr-engine-query/src/main/kotlin/org/vitrivr/engine/query/operators/transform/lookup/ObjectFieldLookup.kt +++ b/vitrivr-engine-query/src/main/kotlin/org/vitrivr/engine/query/operators/transform/lookup/ObjectFieldLookup.kt @@ -38,7 +38,7 @@ class ObjectFieldLookup(override val input: Operator, private v /* Fetch descriptors for retrievables that should be enriched. */ val descriptors = if (enrich.isNotEmpty()) { - this@ObjectFieldLookup.reader.getAllFor(enrich.keys).associateBy { it.retrievableId!! } + this@ObjectFieldLookup.reader.getAllForRetrievable(enrich.keys).associateBy { it.retrievableId!! } } else { emptyMap() } diff --git a/vitrivr-engine-query/src/main/kotlin/org/vitrivr/engine/query/parsing/QueryParser.kt b/vitrivr-engine-query/src/main/kotlin/org/vitrivr/engine/query/parsing/QueryParser.kt index 4423327c..ea3f4319 100644 --- a/vitrivr-engine-query/src/main/kotlin/org/vitrivr/engine/query/parsing/QueryParser.kt +++ b/vitrivr-engine-query/src/main/kotlin/org/vitrivr/engine/query/parsing/QueryParser.kt @@ -89,38 +89,39 @@ class QueryParser(val schema: Schema) { is RetrievableIdInputData -> { val id = UUID.fromString(input.id) val reader = field.getReader() - val descriptor = reader.getFor(id).firstOrNull() ?: throw IllegalArgumentException("No retrievable with id '$id' present in ${field.fieldName}") + val descriptor = reader.getForRetrievable(id).firstOrNull() ?: throw IllegalArgumentException("No retrievable with id '$id' present in ${field.fieldName}") field.getRetrieverForDescriptor(descriptor, description.context) } - is VectorInputData -> field.getRetrieverForDescriptor(FloatVectorDescriptor(vector = input.data.map { Value.Float(it) }), description.context) + is VectorInputData -> field.getRetrieverForDescriptor(FloatVectorDescriptor(vector = Value.FloatVector(input.data.toFloatArray())), description.context) else -> { /* Is this a boolean sub-field query ? */ if(fieldAndAttributeName.second != null && input.comparison != null){ /* yes */ - val subfield = field.analyser.prototype(field).schema().find { it.name == fieldAndAttributeName.second } ?: throw IllegalArgumentException("Field '${field.fieldName}' does not have a subfield with name '${fieldAndAttributeName.second}'") + val subfield = + field.analyser.prototype(field).layout().find { it.name == fieldAndAttributeName.second } ?: throw IllegalArgumentException("Field '${field.fieldName}' does not have a subfield with name '${fieldAndAttributeName.second}'") /* For now, we support not all input data */ val value = when(input){ is TextInputData -> { - require(subfield.type == Type.STRING){"The given sub-field ${fieldAndAttributeName.first}.${fieldAndAttributeName.second}'s type is ${subfield.type}, which is not the expexted ${Type.STRING}"} + require(subfield.type == Type.String) { "The given sub-field ${fieldAndAttributeName.first}.${fieldAndAttributeName.second}'s type is ${subfield.type}, which is not the expected ${Type.String}" } Value.String(input.data) } is BooleanInputData -> { - require(subfield.type == Type.BOOLEAN){"The given sub-field ${fieldAndAttributeName.first}.${fieldAndAttributeName.second}'s type is ${subfield.type}, which is not the expexted ${Type.BOOLEAN}"} - Value.Boolean(input.value) + require(subfield.type == Type.Boolean) { "The given sub-field ${fieldAndAttributeName.first}.${fieldAndAttributeName.second}'s type is ${subfield.type}, which is not the expected ${Type.Boolean}" } + Value.Boolean(input.data) } is NumericInputData -> { when(subfield.type){ - Type.DOUBLE -> Value.Double(input.value) - Type.INT -> Value.Int(input.value.toInt()) - Type.LONG -> Value.Long(input.value.toLong()) - Type.SHORT -> Value.Short(input.value.toInt().toShort()) - Type.BYTE -> Value.Byte(input.value.toInt().toByte()) - Type.FLOAT -> Value.Float(input.value.toFloat()) + Type.Double -> Value.Double(input.data) + Type.Float -> Value.Float(input.data.toFloat()) + Type.Long -> Value.Long(input.data.toLong()) + Type.Int -> Value.Int(input.data.toInt()) + Type.Short -> Value.Short(input.data.toInt().toShort()) + Type.Byte -> Value.Byte(input.data.toInt().toByte()) else -> throw IllegalArgumentException("Cannot work with NumericInputData $input but non-numerical sub-field $subfield") } } is DateInputData -> { - require(subfield.type == Type.DATETIME){"The given sub-field ${fieldAndAttributeName.first}.${fieldAndAttributeName.second}'s type is ${subfield.type}, which is not the expexted ${Type.DATETIME}"} + require(subfield.type == Type.Datetime) { "The given sub-field ${fieldAndAttributeName.first}.${fieldAndAttributeName.second}'s type is ${subfield.type}, which is not the expected ${Type.Datetime}" } Value.DateTime(input.parseDate()) } else -> throw UnsupportedOperationException("Subfield query for $input is currently not supported") diff --git a/vitrivr-engine-server/build.gradle b/vitrivr-engine-server/build.gradle index b2959089..f050825d 100644 --- a/vitrivr-engine-server/build.gradle +++ b/vitrivr-engine-server/build.gradle @@ -10,6 +10,7 @@ dependencies { api project(':vitrivr-engine-query') api project(':vitrivr-engine-module-features') /* TODO: This dependency is not necessary and only here to facilitate easy testing. */ api project(':vitrivr-engine-module-cottontaildb') /* TODO: This dependency is not necessary and only here to facilitate easy testing. */ + api project(':vitrivr-engine-module-pgvector') /* TODO: This dependency is not necessary and only here to facilitate easy testing. */ api project(':vitrivr-engine-module-fes') /* TODO: This dependency is not necessary and only here to facilitate easy testing. */ /** Clikt & JLine */ diff --git a/vitrivr-engine-server/src/main/kotlin/org/vitrivr/engine/tools/CineastMigrationTool.kt b/vitrivr-engine-server/src/main/kotlin/org/vitrivr/engine/tools/CineastMigrationTool.kt index b5c9c475..0c1d57d6 100644 --- a/vitrivr-engine-server/src/main/kotlin/org/vitrivr/engine/tools/CineastMigrationTool.kt +++ b/vitrivr-engine-server/src/main/kotlin/org/vitrivr/engine/tools/CineastMigrationTool.kt @@ -18,8 +18,6 @@ import org.vitrivr.engine.core.model.descriptor.Descriptor import org.vitrivr.engine.core.model.descriptor.DescriptorId import org.vitrivr.engine.core.model.descriptor.scalar.FloatDescriptor import org.vitrivr.engine.core.model.descriptor.scalar.StringDescriptor -import org.vitrivr.engine.core.model.descriptor.struct.RasterDescriptor -import org.vitrivr.engine.core.model.descriptor.struct.SkeletonDescriptor import org.vitrivr.engine.core.model.descriptor.struct.metadata.MediaDimensionsDescriptor import org.vitrivr.engine.core.model.descriptor.struct.metadata.TemporalMetadataDescriptor import org.vitrivr.engine.core.model.descriptor.struct.metadata.source.FileSourceMetadataDescriptor @@ -31,6 +29,8 @@ import org.vitrivr.engine.core.model.retrievable.Ingested import org.vitrivr.engine.core.model.retrievable.Retrievable import org.vitrivr.engine.core.model.retrievable.RetrievableId import org.vitrivr.engine.core.model.types.Value +import org.vitrivr.engine.module.features.feature.averagecolorraster.RasterDescriptor +import org.vitrivr.engine.module.features.feature.skeleton.SkeletonDescriptor import java.io.File import java.io.FileReader import java.nio.file.Files @@ -87,7 +87,7 @@ data class CineastVectorFeature(override val id: String, val feature: List, return RasterDescriptor( id = DescriptorId.randomUUID(), retrievableId = RetrievableId.fromString(idmap[id]), - hist = hist.map { Value.Float(it) }, - raster = raster.map { Value.Float(it) }, + mapOf( + "hist" to Value.FloatVector(hist.toFloatArray()), + "raster" to Value.FloatVector(raster.toFloatArray()) + ) ) } } @@ -277,8 +281,10 @@ class CineastMigrationTool(val migrationconfigpath: String, val schemaconfigpath val fileMetadataDescriptor = FileSourceMetadataDescriptor( id = DescriptorId.randomUUID(), retrievableId = objectRetrievable.id, - path = Value.String(mobject.path), - size = Value.Long(size), + mapOf( + "path" to Value.String(mobject.path), + "size" to Value.Long(size), + ) ) filemetadatawriter.add(fileMetadataDescriptor) retrievableWriter.add(objectRetrievable) @@ -326,8 +332,10 @@ class CineastMigrationTool(val migrationconfigpath: String, val schemaconfigpath val dimensionsDescriptor = MediaDimensionsDescriptor( id = DescriptorId.randomUUID(), retrievableId = RetrievableId.fromString(retrievableId), - width = Value.Int(width), - height = Value.Int(height) + mapOf( + "width" to Value.Int(width), + "height" to Value.Int(height) + ) ) mediadimensionswriter.add(dimensionsDescriptor) } @@ -382,8 +390,11 @@ class CineastMigrationTool(val migrationconfigpath: String, val schemaconfigpath val temporalMetadataDescriptor = TemporalMetadataDescriptor( id = DescriptorId.randomUUID(), retrievableId = ingested.id, - startNs = Value.Long(segment.segmentstartabs.toLong() * 1000 * 1000 * 1000), - endNs = Value.Long(segment.segmentendabs.toLong() * 1000 * 1000 * 1000), + mapOf( + "start" to Value.Long(segment.segmentstartabs.toLong() * 1000 * 1000 * 1000), + "end" to Value.Long(segment.segmentendabs.toLong() * 1000 * 1000 * 1000), + ) + ) ingestedList.add(ingested)