Feature: 3D Renderer and Preview Export of Thumbnails

.github/workflows/ci.yml

@@ -29,7 +29,10 @@ jobs:
 
     # Start actual job.
     steps:
-      - uses: actions/checkout@v4
+      - name: Set up Xvfb
+        run: Xvfb :1 -ac -screen 0 1024x768x24 &
+      - name: Checkout code
+        uses: actions/checkout@v4
       - name: Cottontail DB connection test
         run: nc -zv 127.0.0.1 1865
       - name: PostgreSQL connection test
@@ -50,4 +53,9 @@ jobs:
         run: ./gradlew test --info
       - name: Test with gradle windows
         if: matrix.os == 'windows-latest'
-        run: ./gradlew test --info
+        run: ./gradlew test --info
+      - name: Publish Test Report
+        uses: mikepenz/action-junit-report@v4
+        if: success() || failure() # always run even if the previous step fails
+        with:
+          report_paths: '**/build/test-results/test/TEST-*.xml'

config-ingestion.json

@@ -0,0 +1,120 @@
+{
+  "schema": "vitrivr",
+  "context": {
+    "contentFactory": "InMemoryContentFactory",
+    "resolverName": "disk",
+    "local": {
+      "enumerator": {
+        "path": "./path/to/videos",
+        "depth": "1"
+      },
+      "thumbs": {
+        "maxSideResolution": "350",
+        "mimeType": "JPG"
+      },
+      "filter": {
+        "type": "SOURCE:VIDEO"
+      }
+    }
+  },
+  "operators": {
+    "enumerator": {
+      "type": "ENUMERATOR",
+      "factory": "FileSystemEnumerator",
+      "mediaTypes": [
+        "VIDEO"
+      ]
+    },
+    "decoder": {
+      "type": "DECODER",
+      "factory": "VideoDecoder"
+    },
+    "selector": {
+      "type": "TRANSFORMER",
+      "factory": "LastContentAggregator"
+    },
+    "avgColor": {
+      "type": "EXTRACTOR",
+      "fieldName": "averagecolor"
+    },
+    "file_metadata": {
+      "type": "EXTRACTOR",
+      "fieldName": "file"
+    },
+    "time_metadata": {
+      "type": "EXTRACTOR",
+      "fieldName": "time"
+    },
+    "video_metadata": {
+      "type": "EXTRACTOR",
+      "fieldName": "video"
+    },
+    "thumbs": {
+      "type": "EXPORTER",
+      "exporterName": "thumbnail"
+    },
+    "filter": {
+      "type": "TRANSFORMER",
+      "factory": "TypeFilterTransformer"
+    }
+  },
+  "operations": {
+    "enumerator": {
+      "operator": "enumerator"
+    },
+    "decoder": {
+      "operator": "decoder",
+      "inputs": [
+        "enumerator"
+      ]
+    },
+    "selector": {
+      "operator": "selector",
+      "inputs": [
+        "decoder"
+      ]
+    },
+    "averagecolor": {
+      "operator": "avgColor",
+      "inputs": [
+        "selector"
+      ]
+    },
+    "thumbnails": {
+      "operator": "thumbs",
+      "inputs": [
+        "selector"
+      ]
+    },
+    "time_metadata": {
+      "operator": "time_metadata",
+      "inputs": [
+        "selector"
+      ]
+    },
+    "filter": {
+      "operator": "filter",
+      "inputs": [
+        "averagecolor",
+        "thumbnails",
+        "time_metadata"
+      ],
+      "merge": "COMBINE"
+    },
+    "video_metadata": {
+      "operator": "video_metadata",
+      "inputs": [
+        "filter"
+      ]
+    },
+    "file_metadata": {
+      "operator": "file_metadata",
+      "inputs": [
+        "video_metadata"
+      ]
+    }
+  },
+  "output": [
+    "file_metadata"
+  ]
+}

config-schema.json

@@ -0,0 +1,66 @@
+{
+  "schemas": {
+    "vitrivr": {
+      "connection": {
+        "database": "CottontailConnectionProvider",
+        "parameters": {
+          "Host": "127.0.0.1",
+          "port": "1865"
+        }
+      },
+      "fields": {
+        "averagecolor": {
+          "factory": "AverageColor"
+        },
+        "file": {
+          "factory": "FileSourceMetadata"
+        },
+        "time": {
+          "factory": "TemporalMetadata"
+        },
+        "video": {
+          "factory": "VideoSourceMetadata"
+        }
+      },
+      "resolvers": {
+        "disk": {
+          "factory": "DiskResolver",
+          "parameters": {
+            "location": "./thumbnails/vitrivr",
+            "mimeType": "GIF"
+          }
+        }
+      },
+      "exporters": {
+        "thumbnail": {
+          "factory": "ThumbnailExporter",
+          "resolverName": "disk",
+          "parameters": {
+            "maxSideResolution": "400",
+            "mimeType": "JPG"
+          }
+        },
+        "preview": {
+          "factory": "ModelPreviewExporter",
+          "resolverName": "disk",
+          "parameters": {
+            "maxSideResolution": "400",
+            "mimeType": "GLTF",
+            "distance": "1",
+            "format": "jpg",
+            "views": "4"
+          }
+        }
+      },
+      "extractionPipelines": {
+        "ingestion": {
+          "path": "./config-ingestion.json"
+        },
+        "preview": {
+          "path": "./config-ingestion-preview-mesh.json"
+        }
+      }
+    }
+  }
+}
+

gradle.properties

@@ -13,12 +13,13 @@ version_grpc=1.66.0
 version_kotlin=1.9.25
 version_kotlinx_coroutines=1.7.3
 version_kotlinx_serialization=1.6.2
-version_kotlinx_datetime=0.6.1
-version_kotlinlogging = 7.0.0
+version_kotlinx_datetime=0.4.1
+version_kotlinlogging = 5.1.0
+version_lwjgl=3.3.4
 version_log4j2=2.23.1
 version_metadataextractor=2.19.0
 version_picnic=0.7.0
-version_protobuf=4.27.3
+version_protobuf=3.25.4
 version_scrimage=4.2.0
 version_slf4j=2.0.16
 version_jogl=2.3.2

vitrivr-engine-core/build.gradle

@@ -5,9 +5,6 @@ plugins {
 }
 
 dependencies {
-    /** JOML dependencies for 3D mesh support. */
-    implementation group: 'org.joml', name: 'joml', version: version_joml
-
     /** dependencies for exif metadata extraction. */
     implementation group: 'com.drewnoakes', name: 'metadata-extractor', version: version_metadataextractor
     implementation group: 'io.javalin.community.openapi', name: 'javalin-openapi-plugin', version: version_javalin

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/HSVColorContainer.kt

@@ -10,7 +10,7 @@ import java.awt.color.ColorSpace
  * @version 1.0.0
  */
 @JvmInline
-value class HSVColorContainer constructor(private val hsv: FloatArray) {
+value class HSVColorContainer(private val hsv: FloatArray) {
 
     init {
         require(this.hsv.size == 3) { "HSVFloatColorContainer must have exactly 3 elements." }

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/LabColorContainer.kt

@@ -10,7 +10,7 @@ import java.awt.color.ColorSpace
  * @version 1.0.0
  */
 @JvmInline
-value class LabColorContainer constructor(private val lab: FloatArray) {
+value class LabColorContainer(private val lab: FloatArray) {
     init {
         require(this.lab.size == 3) { "LabColorContainer must have exactly 3 elements." }
     }

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBColorContainer.kt

@@ -13,7 +13,7 @@ import kotlin.math.sqrt
  * @version 1.0.0
  */
 @JvmInline
-value class RGBColorContainer constructor(private val rgb: FloatArray) {
+value class RGBColorContainer(private val rgb: FloatArray) {
 
     init {
         require(this.rgb.size == 4) { "RGBFloatColorContainer must have exactly 4 elements." }

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/XYZColorContainer.kt

@@ -10,7 +10,7 @@ import java.awt.color.ColorSpace
  * @version 1.0.0
  */
 @JvmInline
-value class XYZColorContainer constructor(private val xyz: FloatArray) {
+value class XYZColorContainer(private val xyz: FloatArray) {
 
     constructor(x: Float, y: Float, z: Float) : this(floatArrayOf(x, y, z))
     constructor(x: Double, y: Double, z: Double) : this(floatArrayOf(x.toFloat(), y.toFloat(), z.toFloat()))

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/YCbCrColorContainer.kt

@@ -9,7 +9,7 @@ import java.awt.color.ColorSpace
  * @version 1.0.0
  */
 @JvmInline
-value class YCbCrColorContainer constructor(private val ycbcr: FloatArray) {
+value class YCbCrColorContainer(private val ycbcr: FloatArray) {
     init {
         require(this.ycbcr.size == 3) { "YCbCrColorContainer must have exactly 3 elements." }
     }

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/element/Model3DContent.kt

@@ -1,15 +1,15 @@
 package org.vitrivr.engine.core.model.content.element
 
 import org.vitrivr.engine.core.model.content.ContentType
-import org.vitrivr.engine.core.model.mesh.Model3D
+import org.vitrivr.engine.core.model.mesh.texturemodel.Model3d
 
 /**
  * A 3D [ContentElement].
  *
  * @author Rahel Arnold
  * @version 1.0.0
  */
-interface Model3DContent: ContentElement<Model3D>{
+interface Model3DContent : ContentElement<Model3d> {
     /** The [ContentType] of a [Model3DContent] is always [ContentType.MESH]. */
 
     override val type: ContentType

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/factory/CachedContentFactory.kt

@@ -9,7 +9,7 @@ import org.vitrivr.engine.core.model.content.impl.cache.CachedContent
 import org.vitrivr.engine.core.model.content.impl.cache.CachedImageContent
 import org.vitrivr.engine.core.model.content.impl.cache.CachedTextContent
 import org.vitrivr.engine.core.model.content.impl.memory.InMemoryMeshContent
-import org.vitrivr.engine.core.model.mesh.Model3D
+import org.vitrivr.engine.core.model.mesh.texturemodel.Model3d
 import org.vitrivr.engine.core.model.metamodel.Schema
 import java.awt.image.BufferedImage
 import java.io.IOException
@@ -118,9 +118,9 @@ class CachedContentFactory : ContentFactoriesFactory {
             return content
         }
 
-        override fun newMeshContent(model3D: Model3D): Model3DContent {
+        override fun newMeshContent(model3d: Model3d): Model3DContent {
             check(!this.closed) { "CachedContentFactory has been closed." }
-            val content = InMemoryMeshContent(model3D) /* TODO: Caching. */
+            val content = InMemoryMeshContent(model3d) /* TODO: Caching. */
             logger.warn { "Caching of MeshContent is not yet implemented. Using in-memory content instead." }
             return content
         }

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/factory/ContentFactory.kt

@@ -4,7 +4,7 @@ import org.vitrivr.engine.core.model.content.element.AudioContent
 import org.vitrivr.engine.core.model.content.element.ImageContent
 import org.vitrivr.engine.core.model.content.element.Model3DContent
 import org.vitrivr.engine.core.model.content.element.TextContent
-import org.vitrivr.engine.core.model.mesh.Model3D
+import org.vitrivr.engine.core.model.mesh.texturemodel.Model3d
 import java.awt.image.BufferedImage
 import java.nio.ShortBuffer
 
@@ -15,5 +15,5 @@ interface ContentFactory {
 
     fun newTextContent(text: String): TextContent
 
-    fun newMeshContent(model3D: Model3D): Model3DContent
+    fun newMeshContent(model3d: Model3d): Model3DContent
 }

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/factory/InMemoryContentFactory.kt

@@ -8,7 +8,7 @@ import org.vitrivr.engine.core.model.content.impl.memory.InMemoryAudioContent
 import org.vitrivr.engine.core.model.content.impl.memory.InMemoryImageContent
 import org.vitrivr.engine.core.model.content.impl.memory.InMemoryMeshContent
 import org.vitrivr.engine.core.model.content.impl.memory.InMemoryTextContent
-import org.vitrivr.engine.core.model.mesh.Model3D
+import org.vitrivr.engine.core.model.mesh.texturemodel.Model3d
 import org.vitrivr.engine.core.model.metamodel.Schema
 import java.awt.image.BufferedImage
 import java.nio.ShortBuffer
@@ -25,6 +25,6 @@ class InMemoryContentFactory : ContentFactoriesFactory {
         override fun newImageContent(bufferedImage: BufferedImage) = InMemoryImageContent(bufferedImage)
         override fun newAudioContent(channels: Short, sampleRate: Int, audio: ShortBuffer) = InMemoryAudioContent(channels, sampleRate, audio)
         override fun newTextContent(text: String): TextContent = InMemoryTextContent(text)
-        override fun newMeshContent(model3D: Model3D): Model3DContent = InMemoryMeshContent(model3D)
+        override fun newMeshContent(model3d: Model3d): Model3DContent = InMemoryMeshContent(model3d)
     }
 }

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/impl/memory/InMemoryMeshContent.kt → vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/impl/memory/InMemoryMesh3DContent.kt

@@ -3,8 +3,7 @@ package org.vitrivr.engine.core.model.content.impl.memory
 import org.vitrivr.engine.core.model.content.element.ContentId
 import org.vitrivr.engine.core.model.content.element.ImageContent
 import org.vitrivr.engine.core.model.content.element.Model3DContent
-import org.vitrivr.engine.core.model.mesh.Model3D
-import java.util.*
+import org.vitrivr.engine.core.model.mesh.texturemodel.Model3d
 
 /**
  * A naive in-memory implementation of the [ImageContent] interface.
@@ -14,4 +13,4 @@ import java.util.*
  * @author Luca Rossetto.
  * @version 1.0.0
  */
-data class InMemoryMeshContent(override val content: Model3D, override val id: ContentId = ContentId.randomUUID()) : Model3DContent
+data class InMemoryMeshContent(override val content: Model3d, override val id: ContentId = ContentId.randomUUID()) : Model3DContent