Skip to content

Commit

Permalink
Merge pull request #87 from vitrivr/feature/3d-preview
Browse files Browse the repository at this point in the history
Feature: 3D Renderer and Preview Export of Thumbnails
  • Loading branch information
rahelarnold98 authored Aug 27, 2024
2 parents 92a3331 + 1247b9c commit 9fee7e5
Show file tree
Hide file tree
Showing 107 changed files with 7,537 additions and 909 deletions.
12 changes: 10 additions & 2 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,10 @@ jobs:

# Start actual job.
steps:
- uses: actions/checkout@v4
- name: Set up Xvfb
run: Xvfb :1 -ac -screen 0 1024x768x24 &
- name: Checkout code
uses: actions/checkout@v4
- name: Cottontail DB connection test
run: nc -zv 127.0.0.1 1865
- name: PostgreSQL connection test
Expand All @@ -50,4 +53,9 @@ jobs:
run: ./gradlew test --info
- name: Test with gradle windows
if: matrix.os == 'windows-latest'
run: ./gradlew test --info
run: ./gradlew test --info
- name: Publish Test Report
uses: mikepenz/action-junit-report@v4
if: success() || failure() # always run even if the previous step fails
with:
report_paths: '**/build/test-results/test/TEST-*.xml'
120 changes: 120 additions & 0 deletions config-ingestion.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
{
"schema": "vitrivr",
"context": {
"contentFactory": "InMemoryContentFactory",
"resolverName": "disk",
"local": {
"enumerator": {
"path": "./path/to/videos",
"depth": "1"
},
"thumbs": {
"maxSideResolution": "350",
"mimeType": "JPG"
},
"filter": {
"type": "SOURCE:VIDEO"
}
}
},
"operators": {
"enumerator": {
"type": "ENUMERATOR",
"factory": "FileSystemEnumerator",
"mediaTypes": [
"VIDEO"
]
},
"decoder": {
"type": "DECODER",
"factory": "VideoDecoder"
},
"selector": {
"type": "TRANSFORMER",
"factory": "LastContentAggregator"
},
"avgColor": {
"type": "EXTRACTOR",
"fieldName": "averagecolor"
},
"file_metadata": {
"type": "EXTRACTOR",
"fieldName": "file"
},
"time_metadata": {
"type": "EXTRACTOR",
"fieldName": "time"
},
"video_metadata": {
"type": "EXTRACTOR",
"fieldName": "video"
},
"thumbs": {
"type": "EXPORTER",
"exporterName": "thumbnail"
},
"filter": {
"type": "TRANSFORMER",
"factory": "TypeFilterTransformer"
}
},
"operations": {
"enumerator": {
"operator": "enumerator"
},
"decoder": {
"operator": "decoder",
"inputs": [
"enumerator"
]
},
"selector": {
"operator": "selector",
"inputs": [
"decoder"
]
},
"averagecolor": {
"operator": "avgColor",
"inputs": [
"selector"
]
},
"thumbnails": {
"operator": "thumbs",
"inputs": [
"selector"
]
},
"time_metadata": {
"operator": "time_metadata",
"inputs": [
"selector"
]
},
"filter": {
"operator": "filter",
"inputs": [
"averagecolor",
"thumbnails",
"time_metadata"
],
"merge": "COMBINE"
},
"video_metadata": {
"operator": "video_metadata",
"inputs": [
"filter"
]
},
"file_metadata": {
"operator": "file_metadata",
"inputs": [
"video_metadata"
]
}
},
"output": [
"file_metadata"
]
}
66 changes: 66 additions & 0 deletions config-schema.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
{
"schemas": {
"vitrivr": {
"connection": {
"database": "CottontailConnectionProvider",
"parameters": {
"Host": "127.0.0.1",
"port": "1865"
}
},
"fields": {
"averagecolor": {
"factory": "AverageColor"
},
"file": {
"factory": "FileSourceMetadata"
},
"time": {
"factory": "TemporalMetadata"
},
"video": {
"factory": "VideoSourceMetadata"
}
},
"resolvers": {
"disk": {
"factory": "DiskResolver",
"parameters": {
"location": "./thumbnails/vitrivr",
"mimeType": "GIF"
}
}
},
"exporters": {
"thumbnail": {
"factory": "ThumbnailExporter",
"resolverName": "disk",
"parameters": {
"maxSideResolution": "400",
"mimeType": "JPG"
}
},
"preview": {
"factory": "ModelPreviewExporter",
"resolverName": "disk",
"parameters": {
"maxSideResolution": "400",
"mimeType": "GLTF",
"distance": "1",
"format": "jpg",
"views": "4"
}
}
},
"extractionPipelines": {
"ingestion": {
"path": "./config-ingestion.json"
},
"preview": {
"path": "./config-ingestion-preview-mesh.json"
}
}
}
}
}

5 changes: 3 additions & 2 deletions gradle.properties
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,9 @@ version_grpc=1.66.0
version_kotlin=1.9.25
version_kotlinx_coroutines=1.7.3
version_kotlinx_serialization=1.6.2
version_kotlinx_datetime=0.6.1
version_kotlinlogging = 7.0.0
version_kotlinx_datetime=0.4.1
version_kotlinlogging = 5.1.0
version_lwjgl=3.3.4
version_log4j2=2.23.1
version_metadataextractor=2.19.0
version_nightmonkeys=1.0.0
Expand Down
3 changes: 0 additions & 3 deletions vitrivr-engine-core/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,6 @@ plugins {
}

dependencies {
/** JOML dependencies for 3D mesh support. */
implementation group: 'org.joml', name: 'joml', version: version_joml

/** dependencies for exif metadata extraction. */
implementation group: 'com.drewnoakes', name: 'metadata-extractor', version: version_metadataextractor
implementation group: 'io.javalin.community.openapi', name: 'javalin-openapi-plugin', version: version_javalin
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import java.awt.color.ColorSpace
* @version 1.0.0
*/
@JvmInline
value class HSVColorContainer constructor(private val hsv: FloatArray) {
value class HSVColorContainer(private val hsv: FloatArray) {

init {
require(this.hsv.size == 3) { "HSVFloatColorContainer must have exactly 3 elements." }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import java.awt.color.ColorSpace
* @version 1.0.0
*/
@JvmInline
value class LabColorContainer constructor(private val lab: FloatArray) {
value class LabColorContainer(private val lab: FloatArray) {
init {
require(this.lab.size == 3) { "LabColorContainer must have exactly 3 elements." }
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import kotlin.math.sqrt
* @version 1.0.0
*/
@JvmInline
value class RGBColorContainer constructor(private val rgb: FloatArray) {
value class RGBColorContainer(private val rgb: FloatArray) {

init {
require(this.rgb.size == 4) { "RGBFloatColorContainer must have exactly 4 elements." }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import java.awt.color.ColorSpace
* @version 1.0.0
*/
@JvmInline
value class XYZColorContainer constructor(private val xyz: FloatArray) {
value class XYZColorContainer(private val xyz: FloatArray) {

constructor(x: Float, y: Float, z: Float) : this(floatArrayOf(x, y, z))
constructor(x: Double, y: Double, z: Double) : this(floatArrayOf(x.toFloat(), y.toFloat(), z.toFloat()))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import java.awt.color.ColorSpace
* @version 1.0.0
*/
@JvmInline
value class YCbCrColorContainer constructor(private val ycbcr: FloatArray) {
value class YCbCrColorContainer(private val ycbcr: FloatArray) {
init {
require(this.ycbcr.size == 3) { "YCbCrColorContainer must have exactly 3 elements." }
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
package org.vitrivr.engine.core.model.content.element

import org.vitrivr.engine.core.model.content.ContentType
import org.vitrivr.engine.core.model.mesh.Model3D
import org.vitrivr.engine.core.model.mesh.texturemodel.Model3d

/**
* A 3D [ContentElement].
*
* @author Rahel Arnold
* @version 1.0.0
*/
interface Model3DContent: ContentElement<Model3D>{
interface Model3DContent : ContentElement<Model3d> {
/** The [ContentType] of a [Model3DContent] is always [ContentType.MESH]. */

override val type: ContentType
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import org.vitrivr.engine.core.model.content.impl.cache.CachedContent
import org.vitrivr.engine.core.model.content.impl.cache.CachedImageContent
import org.vitrivr.engine.core.model.content.impl.cache.CachedTextContent
import org.vitrivr.engine.core.model.content.impl.memory.InMemoryMeshContent
import org.vitrivr.engine.core.model.mesh.Model3D
import org.vitrivr.engine.core.model.mesh.texturemodel.Model3d
import org.vitrivr.engine.core.model.metamodel.Schema
import java.awt.image.BufferedImage
import java.io.IOException
Expand Down Expand Up @@ -118,9 +118,9 @@ class CachedContentFactory : ContentFactoriesFactory {
return content
}

override fun newMeshContent(model3D: Model3D): Model3DContent {
override fun newMeshContent(model3d: Model3d): Model3DContent {
check(!this.closed) { "CachedContentFactory has been closed." }
val content = InMemoryMeshContent(model3D) /* TODO: Caching. */
val content = InMemoryMeshContent(model3d) /* TODO: Caching. */
logger.warn { "Caching of MeshContent is not yet implemented. Using in-memory content instead." }
return content
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import org.vitrivr.engine.core.model.content.element.AudioContent
import org.vitrivr.engine.core.model.content.element.ImageContent
import org.vitrivr.engine.core.model.content.element.Model3DContent
import org.vitrivr.engine.core.model.content.element.TextContent
import org.vitrivr.engine.core.model.mesh.Model3D
import org.vitrivr.engine.core.model.mesh.texturemodel.Model3d
import java.awt.image.BufferedImage
import java.nio.ShortBuffer

Expand All @@ -15,5 +15,5 @@ interface ContentFactory {

fun newTextContent(text: String): TextContent

fun newMeshContent(model3D: Model3D): Model3DContent
fun newMeshContent(model3d: Model3d): Model3DContent
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import org.vitrivr.engine.core.model.content.impl.memory.InMemoryAudioContent
import org.vitrivr.engine.core.model.content.impl.memory.InMemoryImageContent
import org.vitrivr.engine.core.model.content.impl.memory.InMemoryMeshContent
import org.vitrivr.engine.core.model.content.impl.memory.InMemoryTextContent
import org.vitrivr.engine.core.model.mesh.Model3D
import org.vitrivr.engine.core.model.mesh.texturemodel.Model3d
import org.vitrivr.engine.core.model.metamodel.Schema
import java.awt.image.BufferedImage
import java.nio.ShortBuffer
Expand All @@ -25,6 +25,6 @@ class InMemoryContentFactory : ContentFactoriesFactory {
override fun newImageContent(bufferedImage: BufferedImage) = InMemoryImageContent(bufferedImage)
override fun newAudioContent(channels: Short, sampleRate: Int, audio: ShortBuffer) = InMemoryAudioContent(channels, sampleRate, audio)
override fun newTextContent(text: String): TextContent = InMemoryTextContent(text)
override fun newMeshContent(model3D: Model3D): Model3DContent = InMemoryMeshContent(model3D)
override fun newMeshContent(model3d: Model3d): Model3DContent = InMemoryMeshContent(model3d)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,7 @@ package org.vitrivr.engine.core.model.content.impl.memory
import org.vitrivr.engine.core.model.content.element.ContentId
import org.vitrivr.engine.core.model.content.element.ImageContent
import org.vitrivr.engine.core.model.content.element.Model3DContent
import org.vitrivr.engine.core.model.mesh.Model3D
import java.util.*
import org.vitrivr.engine.core.model.mesh.texturemodel.Model3d

/**
* A naive in-memory implementation of the [ImageContent] interface.
Expand All @@ -14,4 +13,4 @@ import java.util.*
* @author Luca Rossetto.
* @version 1.0.0
*/
data class InMemoryMeshContent(override val content: Model3D, override val id: ContentId = ContentId.randomUUID()) : Model3DContent
data class InMemoryMeshContent(override val content: Model3d, override val id: ContentId = ContentId.randomUUID()) : Model3DContent

This file was deleted.

Loading

0 comments on commit 9fee7e5

Please sign in to comment.