diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 18adb31a0..61c190f35 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,7 +29,10 @@ jobs: # Start actual job. steps: - - uses: actions/checkout@v4 + - name: Set up Xvfb + run: Xvfb :1 -ac -screen 0 1024x768x24 & + - name: Checkout code + uses: actions/checkout@v4 - name: Cottontail DB connection test run: nc -zv 127.0.0.1 1865 - name: PostgreSQL connection test @@ -50,4 +53,9 @@ jobs: run: ./gradlew test --info - name: Test with gradle windows if: matrix.os == 'windows-latest' - run: ./gradlew test --info \ No newline at end of file + run: ./gradlew test --info + - name: Publish Test Report + uses: mikepenz/action-junit-report@v4 + if: success() || failure() # always run even if the previous step fails + with: + report_paths: '**/build/test-results/test/TEST-*.xml' \ No newline at end of file diff --git a/config-ingestion.json b/config-ingestion.json new file mode 100755 index 000000000..b471b041d --- /dev/null +++ b/config-ingestion.json @@ -0,0 +1,120 @@ +{ + "schema": "vitrivr", + "context": { + "contentFactory": "InMemoryContentFactory", + "resolverName": "disk", + "local": { + "enumerator": { + "path": "./path/to/videos", + "depth": "1" + }, + "thumbs": { + "maxSideResolution": "350", + "mimeType": "JPG" + }, + "filter": { + "type": "SOURCE:VIDEO" + } + } + }, + "operators": { + "enumerator": { + "type": "ENUMERATOR", + "factory": "FileSystemEnumerator", + "mediaTypes": [ + "VIDEO" + ] + }, + "decoder": { + "type": "DECODER", + "factory": "VideoDecoder" + }, + "selector": { + "type": "TRANSFORMER", + "factory": "LastContentAggregator" + }, + "avgColor": { + "type": "EXTRACTOR", + "fieldName": "averagecolor" + }, + "file_metadata": { + "type": "EXTRACTOR", + "fieldName": "file" + }, + "time_metadata": { + "type": "EXTRACTOR", + "fieldName": "time" + }, + "video_metadata": { + "type": "EXTRACTOR", + "fieldName": "video" + }, + "thumbs": { + "type": "EXPORTER", + "exporterName": "thumbnail" + }, + "filter": { + "type": "TRANSFORMER", + "factory": "TypeFilterTransformer" + } + }, + "operations": { + "enumerator": { + "operator": "enumerator" + }, + "decoder": { + "operator": "decoder", + "inputs": [ + "enumerator" + ] + }, + "selector": { + "operator": "selector", + "inputs": [ + "decoder" + ] + }, + "averagecolor": { + "operator": "avgColor", + "inputs": [ + "selector" + ] + }, + "thumbnails": { + "operator": "thumbs", + "inputs": [ + "selector" + ] + }, + "time_metadata": { + "operator": "time_metadata", + "inputs": [ + "selector" + ] + }, + "filter": { + "operator": "filter", + "inputs": [ + "averagecolor", + "thumbnails", + "time_metadata" + ], + "merge": "COMBINE" + }, + "video_metadata": { + "operator": "video_metadata", + "inputs": [ + "filter" + ] + }, + "file_metadata": { + "operator": "file_metadata", + "inputs": [ + "video_metadata" + ] + } + }, + "output": [ + "file_metadata" + ] +} diff --git a/config-schema.json b/config-schema.json new file mode 100755 index 000000000..0c898a45c --- /dev/null +++ b/config-schema.json @@ -0,0 +1,66 @@ +{ + "schemas": { + "vitrivr": { + "connection": { + "database": "CottontailConnectionProvider", + "parameters": { + "Host": "127.0.0.1", + "port": "1865" + } + }, + "fields": { + "averagecolor": { + "factory": "AverageColor" + }, + "file": { + "factory": "FileSourceMetadata" + }, + "time": { + "factory": "TemporalMetadata" + }, + "video": { + "factory": "VideoSourceMetadata" + } + }, + "resolvers": { + "disk": { + "factory": "DiskResolver", + "parameters": { + "location": "./thumbnails/vitrivr", + "mimeType": "GIF" + } + } + }, + "exporters": { + "thumbnail": { + "factory": "ThumbnailExporter", + "resolverName": "disk", + "parameters": { + "maxSideResolution": "400", + "mimeType": "JPG" + } + }, + "preview": { + "factory": "ModelPreviewExporter", + "resolverName": "disk", + "parameters": { + "maxSideResolution": "400", + "mimeType": "GLTF", + "distance": "1", + "format": "jpg", + "views": "4" + } + } + }, + "extractionPipelines": { + "ingestion": { + "path": "./config-ingestion.json" + }, + "preview": { + "path": "./config-ingestion-preview-mesh.json" + } + } + } + } +} + diff --git a/gradle.properties b/gradle.properties index a66a1f5ec..346f01337 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,12 +13,13 @@ version_grpc=1.66.0 version_kotlin=1.9.25 version_kotlinx_coroutines=1.7.3 version_kotlinx_serialization=1.6.2 -version_kotlinx_datetime=0.6.1 -version_kotlinlogging = 7.0.0 +version_kotlinx_datetime=0.4.1 +version_kotlinlogging = 5.1.0 +version_lwjgl=3.3.4 version_log4j2=2.23.1 version_metadataextractor=2.19.0 version_picnic=0.7.0 -version_protobuf=4.27.3 +version_protobuf=3.25.4 version_scrimage=4.2.0 version_slf4j=2.0.16 version_jogl=2.3.2 diff --git a/vitrivr-engine-core/build.gradle b/vitrivr-engine-core/build.gradle index a456d15fc..86c51aa23 100755 --- a/vitrivr-engine-core/build.gradle +++ b/vitrivr-engine-core/build.gradle @@ -5,9 +5,6 @@ plugins { } dependencies { - /** JOML dependencies for 3D mesh support. */ - implementation group: 'org.joml', name: 'joml', version: version_joml - /** dependencies for exif metadata extraction. */ implementation group: 'com.drewnoakes', name: 'metadata-extractor', version: version_metadataextractor implementation group: 'io.javalin.community.openapi', name: 'javalin-openapi-plugin', version: version_javalin diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/HSVColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/HSVColorContainer.kt index 080735fd4..7aaa0d774 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/HSVColorContainer.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/HSVColorContainer.kt @@ -10,7 +10,7 @@ import java.awt.color.ColorSpace * @version 1.0.0 */ @JvmInline -value class HSVColorContainer constructor(private val hsv: FloatArray) { +value class HSVColorContainer(private val hsv: FloatArray) { init { require(this.hsv.size == 3) { "HSVFloatColorContainer must have exactly 3 elements." } diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/LabColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/LabColorContainer.kt index d8027b85c..4c1074dc4 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/LabColorContainer.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/LabColorContainer.kt @@ -10,7 +10,7 @@ import java.awt.color.ColorSpace * @version 1.0.0 */ @JvmInline -value class LabColorContainer constructor(private val lab: FloatArray) { +value class LabColorContainer(private val lab: FloatArray) { init { require(this.lab.size == 3) { "LabColorContainer must have exactly 3 elements." } } diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBColorContainer.kt index 498a6fad9..2335dd1c4 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBColorContainer.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/RGBColorContainer.kt @@ -13,7 +13,7 @@ import kotlin.math.sqrt * @version 1.0.0 */ @JvmInline -value class RGBColorContainer constructor(private val rgb: FloatArray) { +value class RGBColorContainer(private val rgb: FloatArray) { init { require(this.rgb.size == 4) { "RGBFloatColorContainer must have exactly 4 elements." } diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/XYZColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/XYZColorContainer.kt index f6b5b0cd6..3ee5a171e 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/XYZColorContainer.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/XYZColorContainer.kt @@ -10,7 +10,7 @@ import java.awt.color.ColorSpace * @version 1.0.0 */ @JvmInline -value class XYZColorContainer constructor(private val xyz: FloatArray) { +value class XYZColorContainer(private val xyz: FloatArray) { constructor(x: Float, y: Float, z: Float) : this(floatArrayOf(x, y, z)) constructor(x: Double, y: Double, z: Double) : this(floatArrayOf(x.toFloat(), y.toFloat(), z.toFloat())) diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/YCbCrColorContainer.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/YCbCrColorContainer.kt index 5eef673aa..77550cf9b 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/YCbCrColorContainer.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/color/YCbCrColorContainer.kt @@ -9,7 +9,7 @@ import java.awt.color.ColorSpace * @version 1.0.0 */ @JvmInline -value class YCbCrColorContainer constructor(private val ycbcr: FloatArray) { +value class YCbCrColorContainer(private val ycbcr: FloatArray) { init { require(this.ycbcr.size == 3) { "YCbCrColorContainer must have exactly 3 elements." } } diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/element/Model3DContent.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/element/Model3DContent.kt index a2311c74a..831c3a33c 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/element/Model3DContent.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/element/Model3DContent.kt @@ -1,7 +1,7 @@ package org.vitrivr.engine.core.model.content.element import org.vitrivr.engine.core.model.content.ContentType -import org.vitrivr.engine.core.model.mesh.Model3D +import org.vitrivr.engine.core.model.mesh.texturemodel.Model3d /** * A 3D [ContentElement]. @@ -9,7 +9,7 @@ import org.vitrivr.engine.core.model.mesh.Model3D * @author Rahel Arnold * @version 1.0.0 */ -interface Model3DContent: ContentElement{ +interface Model3DContent : ContentElement { /** The [ContentType] of a [Model3DContent] is always [ContentType.MESH]. */ override val type: ContentType diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/factory/CachedContentFactory.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/factory/CachedContentFactory.kt index 13fc7aff0..3efb6e36c 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/factory/CachedContentFactory.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/factory/CachedContentFactory.kt @@ -9,7 +9,7 @@ import org.vitrivr.engine.core.model.content.impl.cache.CachedContent import org.vitrivr.engine.core.model.content.impl.cache.CachedImageContent import org.vitrivr.engine.core.model.content.impl.cache.CachedTextContent import org.vitrivr.engine.core.model.content.impl.memory.InMemoryMeshContent -import org.vitrivr.engine.core.model.mesh.Model3D +import org.vitrivr.engine.core.model.mesh.texturemodel.Model3d import org.vitrivr.engine.core.model.metamodel.Schema import java.awt.image.BufferedImage import java.io.IOException @@ -118,9 +118,9 @@ class CachedContentFactory : ContentFactoriesFactory { return content } - override fun newMeshContent(model3D: Model3D): Model3DContent { + override fun newMeshContent(model3d: Model3d): Model3DContent { check(!this.closed) { "CachedContentFactory has been closed." } - val content = InMemoryMeshContent(model3D) /* TODO: Caching. */ + val content = InMemoryMeshContent(model3d) /* TODO: Caching. */ logger.warn { "Caching of MeshContent is not yet implemented. Using in-memory content instead." } return content } diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/factory/ContentFactory.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/factory/ContentFactory.kt index 35600e8f2..1b0a3f758 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/factory/ContentFactory.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/factory/ContentFactory.kt @@ -4,7 +4,7 @@ import org.vitrivr.engine.core.model.content.element.AudioContent import org.vitrivr.engine.core.model.content.element.ImageContent import org.vitrivr.engine.core.model.content.element.Model3DContent import org.vitrivr.engine.core.model.content.element.TextContent -import org.vitrivr.engine.core.model.mesh.Model3D +import org.vitrivr.engine.core.model.mesh.texturemodel.Model3d import java.awt.image.BufferedImage import java.nio.ShortBuffer @@ -15,5 +15,5 @@ interface ContentFactory { fun newTextContent(text: String): TextContent - fun newMeshContent(model3D: Model3D): Model3DContent + fun newMeshContent(model3d: Model3d): Model3DContent } \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/factory/InMemoryContentFactory.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/factory/InMemoryContentFactory.kt index a3e5a8327..2c40aad83 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/factory/InMemoryContentFactory.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/factory/InMemoryContentFactory.kt @@ -8,7 +8,7 @@ import org.vitrivr.engine.core.model.content.impl.memory.InMemoryAudioContent import org.vitrivr.engine.core.model.content.impl.memory.InMemoryImageContent import org.vitrivr.engine.core.model.content.impl.memory.InMemoryMeshContent import org.vitrivr.engine.core.model.content.impl.memory.InMemoryTextContent -import org.vitrivr.engine.core.model.mesh.Model3D +import org.vitrivr.engine.core.model.mesh.texturemodel.Model3d import org.vitrivr.engine.core.model.metamodel.Schema import java.awt.image.BufferedImage import java.nio.ShortBuffer @@ -25,6 +25,6 @@ class InMemoryContentFactory : ContentFactoriesFactory { override fun newImageContent(bufferedImage: BufferedImage) = InMemoryImageContent(bufferedImage) override fun newAudioContent(channels: Short, sampleRate: Int, audio: ShortBuffer) = InMemoryAudioContent(channels, sampleRate, audio) override fun newTextContent(text: String): TextContent = InMemoryTextContent(text) - override fun newMeshContent(model3D: Model3D): Model3DContent = InMemoryMeshContent(model3D) + override fun newMeshContent(model3d: Model3d): Model3DContent = InMemoryMeshContent(model3d) } } diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/impl/memory/InMemoryMeshContent.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/impl/memory/InMemoryMesh3DContent.kt similarity index 80% rename from vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/impl/memory/InMemoryMeshContent.kt rename to vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/impl/memory/InMemoryMesh3DContent.kt index 025aeb675..6170e9902 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/impl/memory/InMemoryMeshContent.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/content/impl/memory/InMemoryMesh3DContent.kt @@ -3,8 +3,7 @@ package org.vitrivr.engine.core.model.content.impl.memory import org.vitrivr.engine.core.model.content.element.ContentId import org.vitrivr.engine.core.model.content.element.ImageContent import org.vitrivr.engine.core.model.content.element.Model3DContent -import org.vitrivr.engine.core.model.mesh.Model3D -import java.util.* +import org.vitrivr.engine.core.model.mesh.texturemodel.Model3d /** * A naive in-memory implementation of the [ImageContent] interface. @@ -14,4 +13,4 @@ import java.util.* * @author Luca Rossetto. * @version 1.0.0 */ -data class InMemoryMeshContent(override val content: Model3D, override val id: ContentId = ContentId.randomUUID()) : Model3DContent +data class InMemoryMeshContent(override val content: Model3d, override val id: ContentId = ContentId.randomUUID()) : Model3DContent diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/Material.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/Material.kt deleted file mode 100644 index bf109565f..000000000 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/Material.kt +++ /dev/null @@ -1,28 +0,0 @@ -package org.vitrivr.engine.core.model.mesh - -import org.joml.Vector4f -import java.util.* - -/** - * The Material contains all meshes and the texture that are drawn with on the meshes. - * Further, it contains the diffuse color of the material. - */ -data class Material(val meshes: MutableList = mutableListOf(), var texture: Texture = Texture(), var diffuseColor: Vector4f = DEFAULT_COLOR) { - /** - * Empty material that can be used as a placeholder. - */ - companion object { - val DEFAULT_COLOR: Vector4f = Vector4f(0.0f, 0.0f, 0.0f, 1.0f) - } - - /** - * @return A [MinimalBoundingBox] which encloses all [MinimalBoundingBoxes][Mesh.getMinimalBoundingBox] from containing meshes. - */ - fun getMinimalBoundingBox(): MinimalBoundingBox { - val mmb = MinimalBoundingBox() - for (mesh in meshes) { - mmb.merge(mesh.minimalBoundingBox) - } - return mmb - } -} diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/Mesh.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/Mesh.kt deleted file mode 100644 index a937639be..000000000 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/Mesh.kt +++ /dev/null @@ -1,200 +0,0 @@ -package org.vitrivr.engine.core.model.mesh - -import org.joml.Vector3f -import org.joml.Vector3i -import org.vitrivr.engine.core.model.mesh.Mesh.Face -import kotlin.math.sign -import kotlin.math.sqrt - -/** - * The [Mesh] is the geometric representation of a 3D model. It contains the vertices, faces, normals, and - * texture coordinates. It also constructs the face normals and the minimal bounding box. - * - * @version 1.0.0 - * @author Raphael Waltenspuel - * @author Rahel Arnold - * @author Ralph Gasser - */ -data class Mesh( - var id: String? = null, - - /** Array of all vertex positions in the mesh. */ - val vertexPositions: List, - - /** Array of all vertex normals in the mesh. */ - val vertexNormals: List, - - /** Array of all texture coordinates in the mesh. */ - val textureCoordinates: List, - - /** An [IntArray] containing all indexes of a [Face]. */ - val faceIndexes: List, -) { - companion object { - - /** - * Constructs a [Mesh] from a bunch of [FloatArray] and [IntArray]s. - * - * For all arrays provided, triples of values will be interpreted as components of coordiantes belonging together. - * For example: 0, 2, 3, 5, 2, 5 == [0, 2, 3], [5, 2, 5] - * - * @param vertexPositions [FloatArray] of [Vertex] positions. - * @param vertexNormals [FloatArray] of [Vertex] normals - * @param textureCoordinates [FloatArray] of texture coordinates. - * @param faceIndexes [IntArray] Array of face indexes. - */ - fun of(vertexPositions: FloatArray, vertexNormals: FloatArray?, textureCoordinates: FloatArray, faceIndexes: IntArray): Mesh { - val positions = vertexPositions.asSequence().windowed(3, 3).map { Vector3f(it[0], it[1], it[2]) }.toList() - val normals = vertexNormals?.asSequence()?.windowed(3, 3)?.map { Vector3f(it[0], it[1], it[2]) }?.toList() - ?: List(positions.size) { Vector3f(0.0f, 0.0f, 0.0f) } - val coordinates = textureCoordinates.asSequence().windowed(3, 3).map { Vector3f(it[0], it[1], it[2]) }.toList() - val indexes = faceIndexes.asSequence().windowed(3, 3).map { Vector3i(it[0], it[1], it[2]) }.toList() - return Mesh(vertexPositions = positions, vertexNormals = normals, textureCoordinates = coordinates, faceIndexes = indexes) - } - } - - init { - /* Perform some santiy checks. */ - require(vertexPositions.size == vertexNormals.size) { "The number of vertex positions and vertex normals must be equal."} - } - - /** Number of all [Vertex] in the [Mesh]. */ - val numberOfVertices: Int - get() = this.vertexPositions.size - - /** Number of all [Face]s in the [Mesh]. */ - val numberOfFaces: Int - get() = this.faceIndexes.size - - /** - * List of all face normal [Vector3f] in the [Mesh]. - * - * The length of the normals describes the area of the face. - * The direction of the normals describes the direction of the face and points outwards. - */ - val faceNormals: List by lazy { - val list = ArrayList(this.numberOfVertices) - for (ic in this.faceIndexes.indices step 3) { - if (ic == faceIndexes.size-2) { - // reached end of loop - break - } - - // Get the three vertices of the face - val v1 = this.vertexPositions[this.faceIndexes[ic].x] - val v2 = this.vertexPositions[this.faceIndexes[ic].y] - val v3 = this.vertexPositions[this.faceIndexes[ic].z] - - // Get the three vertices normals of the face - val vn1 = this.vertexNormals[this.faceIndexes[ic].x] - val vn2 = this.vertexNormals[this.faceIndexes[ic].y] - val vn3 = this.vertexNormals[this.faceIndexes[ic].z] - - // Instance the face normal - val fn = Vector3f(0f, 0f, 0f) - // Calculate the direction of the face normal by averaging the three vertex normals - fn.add(vn1).add(vn2).add(vn3).div(3f).normalize() - // Instance the face area - val fa = Vector3f(0f, 0f, 0f) - // Calculate the area of the face by calculating the cross product of the two edges and dividing by 2 - v2.sub(v1).cross(v3.sub(v1), fa) - fa.div(2f) - // Add the face normal to the list of face normals - list.add(fn.mul(fa.length())) - } - list - } - - /** [MinimalBoundingBox] that encloses this [Mesh]. */ - val minimalBoundingBox: MinimalBoundingBox by lazy { MinimalBoundingBox(this.vertexPositions) } - - /** - * Returns an [Iterator] for all the [Face]s contained in this [Mesh]. - * - * @return [Iterator] of [Face]s - */ - fun faces(): Iterator = object: Iterator { - private var index: Int = 0 - override fun hasNext(): Boolean = this.index <= this@Mesh.faceIndexes.size - override fun next(): Face = Face(this.index++) - } - - /** - * - */ - fun getVertex(index: Int) = Vertex(index) - - /** - * Returns an [Iterator] for all the [Vertex]s contained in this [Mesh]. - * - * @return [Iterator] of [Vertex]s - */ - fun vertices(): Iterator = object: Iterator { - private var index: Int = 0 - override fun hasNext(): Boolean = this.index <= this@Mesh.numberOfVertices - override fun next(): Vertex = Vertex(this.index++) - } - - /** - * A geometric [Vertex] for this [Face] - */ - inner class Vertex(index: Int) { - /** Position of the vertex in 3D space. */ - val position: Vector3f = this@Mesh.vertexPositions[index] - - /** Position of the vertex in 3D space. */ - val normals: Vector3f = this@Mesh.vertexNormals[index] - } - - /** - * A geometric face for this [Mesh]. - */ - inner class Face(index: Int) { - /** The [Vertex] objects that make-up this [Face]. */ - val vertices: List = listOf( - Vertex(this@Mesh.faceIndexes[index].x), - Vertex(this@Mesh.faceIndexes[index].y), - Vertex(this@Mesh.faceIndexes[index].z) - ) - - /** The [Face] normal of this [Face]. */ - val normal: Vector3f = this@Mesh.faceNormals[index] - - /** The centroid for this [Face]. */ - val centroid: Vector3f by lazy { - val centroid = Vector3f(0f, 0f, 0f) - for (vertex in this.vertices) { - centroid.add(vertex.position) - } - centroid.div(3.0f) - centroid - } - - /** The area of this [Face]*/ - val area: Double by lazy { - /* Extract vertices. */ - val v1 = this.vertices[0].position - val v2 = this.vertices[1].position - val v3 = this.vertices[2].position - - /* Generate the edges and sort them in ascending order. */ - val edges: MutableList = ArrayList() - edges.add(Vector3f(v1).sub(v2)) - edges.add(Vector3f(v2).sub(v3)) - edges.add(Vector3f(v3).sub(v1)) - - edges.sortWith{ o1: Vector3f, o2: Vector3f -> - val difference = o1.length() - o2.length() - difference.sign.toInt() - } - - val a = edges[2].length() - val b = edges[1].length() - val c = edges[0].length() - - /* Returns the area of the triangle according to Heron's Formula. */ - val area: Double = 0.25 * sqrt((a + (b + c)) * (c - (a - b)) * (c + (a - b)) * (a + (b - c))) - if (area.isNaN()) { 0.0 } else { area } - } - } -} diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/Model3D.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/Model3D.kt deleted file mode 100644 index d86b19d43..000000000 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/Model3D.kt +++ /dev/null @@ -1,68 +0,0 @@ -package org.vitrivr.engine.core.model.mesh - -import org.joml.Vector3f -import java.util.* - -/** - * This class represents a 3d [Model3D]. It comprises a list of [Entity] objects and a list of [Material] objects. - *The [Entity] objects are used to position and scale the model in the scene. The [Material] objects are used to - * define the appearance of the model. - * - * @version 1.0.0 - * @author Raphael Waltenspuel - * @author Rahel Arnold - * @author Ralph Gasser - */ -data class Model3D( - /** The ID of this [Model3D]. */ - override val id: String, - - /** List of [Entity] objects that define the position and scale of the model. */ - private val entities: MutableList = mutableListOf(), - - /** - * List of [Material] objects that define the appearance of the model. - * Contains all Meshes and Textures that are used by the model. - */ - private val materials: MutableList = mutableListOf() -) : IModel { - /** - * Adds an entity to the model and normalizes the model. - * @param entity Entity to be added. - */ - fun addEntityNorm(entity: Entity) { - val mbb = MinimalBoundingBox() - - for (material in this.materials) { - mbb.merge(material.getMinimalBoundingBox()) - } - - entity.setPosition(mbb.getTranslationToNorm().mul(-1f)) - entity.setScale(mbb.getScalingFactorToNorm()) - entities.add(entity) - } - - /** - * {@inheritDoc} - */ - override fun addEntity(entity: Entity) { - entities.add(entity) - } - - /** - * {@inheritDoc} - */ - override fun getEntities(): List = Collections.unmodifiableList(this.entities) - - /** - * {@inheritDoc} - */ - override fun getMaterials(): List = Collections.unmodifiableList(this.materials) - - /** - * {@inheritDoc} - */ - override fun getAllNormals(): List = this.materials.flatMap { mat -> - mat.meshes.flatMap { mesh -> mesh.vertexNormals } - } -} diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/Texture.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/Texture.kt deleted file mode 100644 index ef719a814..000000000 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/Texture.kt +++ /dev/null @@ -1,28 +0,0 @@ -package org.vitrivr.engine.core.model.mesh - - -import java.io.File -import java.nio.file.Path - -/** - * This class represents a [Texture]. In the context free 3D model, a texture is basically a path to a texture file. - * - * @version 1.0.0 - * @author Raphael Waltenspuel - * @author Rahel Arnold - * @author Ralph Gasser - */ -data class Texture(val path: Path = DEFAULT_TEXTURE) { - companion object { - /** Default texture path. Points to a png with one white pixel with 100% opacity. */ - val DEFAULT_TEXTURE: Path = Path.of("./resources/renderer/lwjgl/models/default/default.png") - } - - /** - * Constructor for the Texture class. - * Sets the texture path to the given texture path. - * - * @param texturePath Path to the texture file. - */ - constructor(texturePath: File) : this(texturePath.toPath()) -} diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/Entity.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/Entity.kt similarity index 50% rename from vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/Entity.kt rename to vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/Entity.kt index 03e73f15d..52ae88c82 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/Entity.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/Entity.kt @@ -1,91 +1,53 @@ -package org.vitrivr.engine.core.model.mesh +package org.vitrivr.engine.core.model.mesh.texturemodel +import org.vitrivr.engine.core.model.mesh.texturemodel.util.types.Vec3f import org.apache.logging.log4j.LogManager -import org.joml.Matrix4f -import org.joml.Quaternionf -import org.joml.Vector3f +import org.apache.logging.log4j.Logger +import org.vitrivr.engine.core.model.mesh.texturemodel.util.types.Matrix4f +import org.vitrivr.engine.core.model.mesh.texturemodel.util.types.Quaternionf /** - * An Entity in the context of a [Model3D] describes a position and scale of a model in the scene. + * An Entity in the context of a [Model3d] describes a position and scale of a model in the scene. * The Entity is composed of a model matrix that is used to transform the model in the scene. * The model matrix is calculated from the position, rotation and scale of the entity. * The Entity influences how the model is rendered in the scene. * It does not change the mesh of the model. * Neither does it change the viewpoint of the camera. */ -class Entity( - /** - * ID of entity. - */ - private val id: String, - /** - * ID of associated model. - */ - private val modelId: String -) { - private val LOGGER = LogManager.getLogger() +class Entity(val id: String, val modelId: String) { /** * Model matrix of entity. * Used to transform the model in the scene. * Calculated from position, rotation and scale. */ - private val modelMatrix: Matrix4f = Matrix4f() + val modelMatrix: Matrix4f = Matrix4f() /** * Position of entity. */ - private val position: Vector3f = Vector3f() + var entityPosition: Vec3f = Vec3f() /** * Rotation of entity. */ - private val rotation: Quaternionf = Quaternionf() + val enttityRotation: Quaternionf = Quaternionf() /** * Scale of entity. */ - private var scale: Float = 1f + var entityScale: Float = 1f - /** - * Constructs a new Entity. - * Defines an associated model and an id. - * With associated model one is able to add new transformations to the Scene [GLScene.addEntity]. - * - * @param id ID of entity. - * @param modelId ID of associated model. - */ init { - this.updateModelMatrix() - } - - /** - * @return Unique ID of entity. - */ - fun getId(): String { - return this.id - } - - /** - * @return ID of the associated model. - */ - fun getModelId(): String { - return this.modelId - } - - /** - * @return Model matrix of entity, describes a rigid transformation of the Model. - */ - fun getModelMatrix(): Matrix4f { - return this.modelMatrix + updateModelMatrix() } /** * Translation values, contained in the ModelMatrix - * @return Translate position of entity in x, y, z. + * @return Translativ position of entity in x, y, z. */ - fun getPosition(): Vector3f { - return this.position + fun getPosition(): Vec3f { + return entityPosition } /** @@ -93,7 +55,7 @@ class Entity( * @return Rotation around x,y,z axes as a quaternion. */ fun getRotation(): Quaternionf { - return this.rotation + return enttityRotation } /** @@ -102,7 +64,7 @@ class Entity( * @return Scale value. */ fun getScale(): Float { - return this.scale + return entityScale } /** @@ -111,16 +73,17 @@ class Entity( * @param y Y coordinate of position. * @param z Z coordinate of position. */ + @Suppress("unused") fun setPosition(x: Float, y: Float, z: Float) { - this.position.set(x, y, z) + entityPosition.set(x, y, z) } /** * Sets translation vector from the origin. * @param position Position of entity. */ - fun setPosition(position: Vector3f) { - this.position.set(position) + fun setPosition(position: Vec3f) { + this.entityPosition.set(position) } /** @@ -131,7 +94,7 @@ class Entity( * @param angle Angle of rotation. */ fun setRotation(x: Float, y: Float, z: Float, angle: Float) { - this.rotation.setAngleAxis(angle, x, y, z) + enttityRotation.fromAxisAngleRad(x, y, z, angle) } /** @@ -139,8 +102,8 @@ class Entity( * @param axis Axis of rotation. * @param angle Angle of rotation. */ - fun setRotation(axis: Vector3f, angle: Float) { - rotation.fromAxisAngleRad(axis, angle) + fun setRotation(axis: Vec3f, angle: Float) { + enttityRotation.fromAxisAngleRad(axis, angle) } /** @@ -149,26 +112,30 @@ class Entity( * @param scale Scale of entity. */ fun setScale(scale: Float) { - this.scale = scale + this.entityScale = scale } /** * Updates the model matrix of the entity. - * @implSpec This has to be called after any transformation. + * This has to be called after any transformation. */ - private fun updateModelMatrix() { - this.modelMatrix.translationRotateScale(this.position, this.rotation, this.scale) + fun updateModelMatrix() { + modelMatrix.translationRotateScale(entityPosition, enttityRotation, entityScale) } /** * Closes the entity. - * Sets the position, rotation to zero and scale to 1. + * Sets the position, rotation to zero and scale to 1. */ fun close() { - this.position.zero() - this.rotation.identity() - this.scale = 1f - this.updateModelMatrix() - LOGGER.trace("Entity {} closed", this.id) + entityPosition.zero() + enttityRotation.identity() + entityScale = 1f + updateModelMatrix() + LOGGER.trace("Entity {} closed", id) + } + + companion object { + private val LOGGER: Logger = LogManager.getLogger(Entity::class.java) } -} \ No newline at end of file +} diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/IModel.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/IModel.kt similarity index 56% rename from vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/IModel.kt rename to vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/IModel.kt index dfd92f428..7c9e60918 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/IModel.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/IModel.kt @@ -1,13 +1,14 @@ -package org.vitrivr.engine.core.model.mesh +package org.vitrivr.engine.core.model.mesh.texturemodel -import org.joml.Vector3f +import org.vitrivr.engine.core.model.mesh.texturemodel.util.types.Vec3f -/** - * - */ interface IModel { - /** The identifier of this [IModel] within the scene. */ - val id: String + + /** + * Returns a list of all entities that are associated with this model. + * @return List of [Entity] objects. + */ + fun getEntities(): List /** * Adds an entity to the model. @@ -16,23 +17,20 @@ interface IModel { fun addEntity(entity: Entity) /** - * Returns a list of all entities that are associated with this model. - * - * @return List of [Entity] objects. + * Returns the id of the model. + * @return ID of the model. */ - fun getEntities(): List + fun getId(): String /** * Returns a list of all materials that are associated with this model. - * * @return List of [Material] objects. */ fun getMaterials(): List /** - * Returns a list of all vertex normals that are associated with this model. - * - * @return List of [Vector3f]. + * Returns a list of all vertices that are associated with this model. + * @return List of [Vec3f] objects. */ - fun getAllNormals(): List -} \ No newline at end of file + fun getAllNormals(): List +} diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/Material.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/Material.kt new file mode 100644 index 000000000..1cbe610e0 --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/Material.kt @@ -0,0 +1,93 @@ +package org.vitrivr.engine.core.model.mesh.texturemodel + +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import org.vitrivr.engine.core.model.mesh.texturemodel.util.MinimalBoundingBox +import org.vitrivr.engine.core.model.mesh.texturemodel.util.types.Vec4f +import java.io.Serializable +import java.util.* + +/** + * The Material contains all meshes and the texture that are drawn with on the meshes. + * Further, it contains the diffuse color of the material. + */ +class Material : Serializable { + + /** + * List of [Mesh] objects that define the appearance of the model. + */ + val materialMeshes: MutableList = ArrayList() + + /** + * Texture that is drawn on all meshes. + */ + var materialTexture: Texture? = Texture() + + /** + * diffuseColor is the color that is drawn on the meshes when no texture is present. + */ + var materialDiffuseColor: Vec4f = DEFAULT_COLOR + + companion object { + private val LOGGER: Logger = LogManager.getLogger() + + /** + * DEFAULT_COLOR is black and 100% opaque. + */ + val DEFAULT_COLOR = Vec4f(0.0f, 0.0f, 0.0f, 1.0f) + + /** + * Empty material that can be used as a placeholder. + */ + val EMPTY = Material() + } + + /** + * @return A MinimalBoundingBox which encloses all MinimalBoundingBoxes from containing meshes. + */ + fun getMinimalBoundingBox(): MinimalBoundingBox { + val mmb = MinimalBoundingBox() + for (mesh in materialMeshes) { + mmb.merge(mesh.getMinimalBoundingBox()) + } + return mmb + } + + /** + * @return an unmodifiable list of meshes. + */ + fun getMeshes(): List = Collections.unmodifiableList(materialMeshes) + + /** + * @param mesh adds a mesh to the material. + */ + fun addMesh(mesh: Mesh) { + materialMeshes.add(mesh) + } + + /** + * @param texture sets the texture to this material. + */ + fun setTexture(texture: Texture) { + this.materialTexture = texture + } + + /** + * @param diffuseColor sets the diffuse color of this material. + */ + fun setDiffuseColor(diffuseColor: Vec4f) { + this.materialDiffuseColor = diffuseColor + } + + /** + * Closes all resources the material uses. + * Calls close on all containing classes. + */ + fun close() { + materialMeshes.forEach(Mesh::close) + materialMeshes.clear() + materialTexture = null + materialDiffuseColor = DEFAULT_COLOR + LOGGER.trace("Closed Material") + } +} diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/Mesh.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/Mesh.kt new file mode 100644 index 000000000..15d5ed5de --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/Mesh.kt @@ -0,0 +1,245 @@ +package org.vitrivr.engine.core.model.mesh.texturemodel + +import org.vitrivr.engine.core.model.mesh.texturemodel.util.types.Vec3f +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import org.vitrivr.engine.core.model.mesh.texturemodel.util.MinimalBoundingBox +import java.io.Serializable +import kotlin.math.sign +import kotlin.math.sqrt + +/** + * The Mesh is the geometric representation of a model. + * It contains the vertices, faces, normals, and texture coordinates. + * It also constructs the face normals and the minimal bounding box. + */ +class Mesh( + /** + * List of all vertices in the mesh. + * The positions are flattened vectors: + * positions[0] = x + * positions[1] = y + * positions[2] = z + * positions[3] = x + * ... + */ + private val positions: FloatArray, + + /** + * List of all vertices normals in the mesh. + */ + val normals: FloatArray?, + + /** + * List of all texture coordinates in the mesh. + */ + private val textureCoords: FloatArray, + + /** + * Flattered list of all vertices ids. + * A three-tuple describes a face: + * e.g. 0, 1, 3, 3, 1, 2, + * face1 = (0, 1, 3) + * face2 = (3, 1, 2) + */ + private val idx: IntArray +) : Serializable { + + companion object { + private val LOGGER: Logger = LogManager.getLogger() + } + + /** + * Number of all vertices in the mesh. + */ + private val numVertices: Int = idx.size + + /** + * ID of the mesh. + */ + var id: String? = null + + /** + * List of all face normals in the mesh. + * The length of the normals describes the area of the face. + * The direction of the normals describes the direction of the face and points outwards. + */ + private val facenormals: MutableList = ArrayList(numVertices / 3) + + /** + * MinimalBoundingBox that encloses the mesh. + */ + val minBoundingBox: MinimalBoundingBox + + init { + // Calculate face normals + for (ic in idx.indices step 3) { + if (normals == null) { + // Add zero vector if there are no vertex normals + facenormals.add(Vec3f(0f, 0f, 0f)) + } else { + // Get the three vertices of the face + val v1 = Vec3f(positions[idx[ic] * 3], positions[idx[ic] * 3 + 1], positions[idx[ic] * 3 + 2]) + val v2 = Vec3f(positions[idx[ic + 1] * 3], positions[idx[ic + 1] * 3 + 1], positions[idx[ic + 1] * 3 + 2]) + val v3 = Vec3f(positions[idx[ic + 2] * 3], positions[idx[ic + 2] * 3 + 1], positions[idx[ic + 2] * 3 + 2]) + + // Get the three vertices normals of the face + val vn1 = Vec3f(normals[idx[ic] * 3], normals[idx[ic] * 3 + 1], normals[idx[ic] * 3 + 2]) + val vn2 = Vec3f(normals[idx[ic + 1] * 3], normals[idx[ic + 1] * 3 + 1], normals[idx[ic + 1] * 3 + 2]) + val vn3 = Vec3f(normals[idx[ic + 2] * 3], normals[idx[ic + 2] * 3 + 1], normals[idx[ic + 2] * 3 + 2]) + + // Instance the face normal + val fn = Vec3f(0f, 0f, 0f) + + // Calculate the direction of the face normal by averaging the three vertex normals + fn.add(vn1).add(vn2).add(vn3).div(3f).normalize() + + // Instance the face area + val fa = Vec3f(0f, 0f, 0f) + + // Calculate the area of the face by calculating the cross product of the two edges and dividing by 2 + v2.subtract(v1).cross(v3.subtract(v1), fa) + fa.div(2f) + + // Add the face normal to the list of face normals + facenormals.add(fn.scale(fa.length())) + } + } + // Calculate the minimal bounding box + minBoundingBox = MinimalBoundingBox(positions) + } + + /** + * @return the number of vertices in the mesh. + */ + fun getNumVertices(): Int = numVertices + + /** + * @return the flattened array of all positions. + */ + fun getPositions(): FloatArray = positions + + /** + * @return the flattened array of all texture coordinates. + */ + fun getTextureCoords(): FloatArray = textureCoords + + fun numberOfFaces(): Int = idx.size / 3 + + /** + * @return the flattened array of all vertices ids. + * A three-tuple describes a face: + * e.g. 0, 1, 3, 3, 1, 2, + * face1 = (0, 1, 3) + * face2 = (3, 1, 2) + */ + fun getIdx(): IntArray = idx + + /** + * @return list containing all face normals. + */ + fun getNormals(): List = facenormals + + /** + * @return the MinimalBoundingBox which contains the scaling factor to norm and the translation to origin (0,0,0). + */ + fun getMinimalBoundingBox(): MinimalBoundingBox = minBoundingBox + + fun getVertex(index: Int) = Vertex(index) + + + fun faces(): Iterator = object: Iterator { + private var index: Int = 0 + override fun hasNext(): Boolean = this.index <= this@Mesh.facenormals.size + override fun next(): Face = Face(this.index++) + } + + fun vertices(): Iterator = object: Iterator { + private var index: Int = 0 + override fun hasNext(): Boolean = this.index <= this@Mesh.normals!!.size + override fun next(): Vertex = Vertex(this.index++) + } + + /** + * @param id sets the id of the mesh. + */ + fun setId(id: Int) { + this.id = id.toString() + } + + /** + * Closes the mesh and releases all resources. + */ + fun close() { + facenormals.clear() + minBoundingBox.close() + id = null + LOGGER.trace("Closing Mesh") + } + + + + /** + * A geometric [Vertex] for this [Face] + */ + inner class Vertex(index: Int) { + /** Position of the vertex in 3D space. */ + val position: Vec3f = Vec3f((this@Mesh.positions.get(index)) *3,this@Mesh.positions[index]*3+1, this@Mesh.positions[index]*3+2) + + /** Position of the vertex in 3D space. */ + val normals: Vec3f = Vec3f((this@Mesh.normals?.get(index)!!) *3,this@Mesh.normals[index]*3+1, this@Mesh.normals[index]*3+2) + } + + /** + * A geometric face for this [Mesh]. + */ + inner class Face(index: Int) { + /** The [Vertex] objects that make-up this [Face]. */ + val vertices: List = listOf( + Vertex(this@Mesh.facenormals[index].x.toInt()), + Vertex(this@Mesh.facenormals[index].y.toInt()), + Vertex(this@Mesh.facenormals[index].z.toInt()) + ) + + /** The [Face] normal of this [Face]. */ + val normal: Vec3f = this@Mesh.facenormals[index] + + /** The centroid for this [Face]. */ + val centroid: Vec3f by lazy { + val centroid = Vec3f(0f, 0f, 0f) + for (vertex in this.vertices) { + centroid.add(vertex.position) + } + centroid.div(3.0f) + centroid + } + + /** The area of this [Face]*/ + val area: Double by lazy { + /* Extract vertices. */ + val v1 = this.vertices[0].position + val v2 = this.vertices[1].position + val v3 = this.vertices[2].position + + /* Generate the edges and sort them in ascending order. */ + val edges: MutableList = ArrayList() + edges.add(Vec3f(v1).subtract(v2)) + edges.add(Vec3f(v2).subtract(v3)) + edges.add(Vec3f(v3).subtract(v1)) + + edges.sortWith{ o1: Vec3f, o2: Vec3f -> + val difference = o1.length() - o2.length() + difference.sign.toInt() + } + + val a = edges[2].length() + val b = edges[1].length() + val c = edges[0].length() + + /* Returns the area of the triangle according to Heron's Formula. */ + val area: Double = 0.25 * sqrt((a + (b + c)) * (c - (a - b)) * (c + (a - b)) * (a + (b - c))) + if (area.isNaN()) { 0.0 } else { area } + } + } + +} \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/Model3d.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/Model3d.kt new file mode 100644 index 000000000..0c5d2aff4 --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/Model3d.kt @@ -0,0 +1,93 @@ +package org.vitrivr.engine.core.model.mesh.texturemodel + +import org.vitrivr.engine.core.model.mesh.texturemodel.util.types.Vec3f +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import org.vitrivr.engine.core.model.mesh.texturemodel.util.MinimalBoundingBox +import java.io.Serializable +import java.util.* +import java.util.function.Consumer + +/** + * This class represents a 3d model that can be rendered by the [Engine]. The 3d3 model is composed of a + * list of [Entity] objects and a list of [Material] objects. The [Entity] objects are used to + * position and scale the 3d model in the scene. The [Material] objects are used to define the + * appearance of the 3d model. + */ +data class Model3d( + /** ID of the 3d model. */ + val modelId: String, + /** + * List of [Material] objects that define the appearance of the model. Contains all Meshes and + * Textures that are used by the 3d model. + */ + val modelMaterials: MutableList +) : IModel, Serializable { + /** List of [Entity] objects that define the position and scale of the 3d model. */ + private val entities: MutableList = ArrayList() + + /** {@inheritDoc} */ + override fun getEntities(): List { + return Collections.unmodifiableList(this.entities) + } + + /** {@inheritDoc} */ + override fun addEntity(entity: Entity) { + entities.add(entity) + } + + /** + * Adds an entity to the model and normalizes the 3d model. + * + * @param entity Entity to be added. + */ + fun addEntityNorm(entity: Entity) { + val mbb = MinimalBoundingBox() + + for (material in this.modelMaterials) { + mbb.merge(material.getMinimalBoundingBox()) + } + + entity.entityPosition = mbb.translationToNorm.mul(-1f) + entity.entityScale = mbb.scalingFactorToNorm + entities.add(entity) + } + + /** {@inheritDoc} */ + override fun getId(): String { + return this.modelId + } + + /** {@inheritDoc} */ + override fun getMaterials(): List { + return Collections.unmodifiableList(this.modelMaterials) + } + + /** {@inheritDoc} */ + override fun getAllNormals(): List { + val normals = ArrayList() + modelMaterials.forEach( + Consumer { m: Material -> + m.materialMeshes.forEach(Consumer { mesh: Mesh -> normals.addAll(mesh.getNormals()) }) + }) + return normals + } + + /** Closes the 3d model and releases all resources. */ + fun close() { + modelMaterials.forEach(Consumer { obj: Material -> obj.close() }) + modelMaterials.clear() + entities.forEach(Consumer { obj: Entity -> obj.close() }) + entities.clear() + LOGGER.trace("Closed model {}", this.modelId) + } + + companion object { + private val LOGGER: Logger = LogManager.getLogger() + + /** + * Empty 3d model that can be used as a placeholder. + */ + val EMPTY = Model3d("EmptyModel", listOf(Material.EMPTY).toMutableList()) + } +} diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/Texture.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/Texture.kt new file mode 100644 index 000000000..ada55ecef --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/Texture.kt @@ -0,0 +1,65 @@ +package org.vitrivr.engine.core.model.mesh.texturemodel + +import java.awt.image.BufferedImage +import java.io.* +import javax.imageio.ImageIO + +/** + * This class represents a texture. + * In the context of a free 3D model, a texture is basically a path to a texture file. + */ +data class Texture( + var texturePath: String? = null, + var textureImage: BufferedImage? = null +) : Serializable { + + companion object { + /** + * Default texture path. + * Points to a png with one white pixel with 100% opacity. + */ + const val DEFAULT_TEXTURE: String = "/renderer/lwjgl/models/default/default.png" + } + + init { + if (texturePath == null && textureImage == null) { + this.textureImage = this.javaClass.getResourceAsStream(DEFAULT_TEXTURE).use { + ImageIO.read(it) + } + } + } + + constructor(texturePath: String) : this(texturePath, null) + + constructor(textureImage: BufferedImage) : this(null, textureImage) + + @Throws(IOException::class) + private fun writeObject(oos: ObjectOutputStream) { + if (this.texturePath != null) { + oos.writeShort(0) + val bytes = this.texturePath!!.toByteArray() + oos.writeInt(bytes.size) + oos.write(bytes) + } else if (this.textureImage != null) { + val baos = ByteArrayOutputStream() + ImageIO.write(this.textureImage, "png", baos) + val bytes = baos.toByteArray() + oos.writeShort(1) + oos.writeInt(bytes.size) + oos.write(bytes) + } + } + + @Throws(IOException::class) + private fun readObject(`in`: ObjectInputStream) { + val mode: Short = `in`.readShort() + val length: Int = `in`.readInt() + val bytes = ByteArray(length) + `in`.readFully(bytes) + if (mode == 0.toShort()) { + this.texturePath = String(bytes) + } else if (mode == 1.toShort()) { + this.textureImage = ImageIO.read(ByteArrayInputStream(bytes)) + } + } +} diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/MinimalBoundingBox.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/util/MinimalBoundingBox.kt similarity index 68% rename from vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/MinimalBoundingBox.kt rename to vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/util/MinimalBoundingBox.kt index 903be0a18..1bf288fe3 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/MinimalBoundingBox.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/util/MinimalBoundingBox.kt @@ -1,51 +1,53 @@ -package org.vitrivr.engine.core.model.mesh +package org.vitrivr.engine.core.model.mesh.texturemodel.util -import org.joml.Vector3f +import org.vitrivr.engine.core.model.mesh.texturemodel.util.types.Vec3f +import java.io.Serializable /** * This class represents a minimal bounding box. It can be generated from a list of vertices. It can be merged with another minimal bounding box. */ -open class MinimalBoundingBox { +class MinimalBoundingBox : Serializable { /** - * Constant for the maximum float value. + * Initial value for the maximum vector. The maximum vector contains the highest (positive if normalized) values for x, y, and z of the bounding box. */ - companion object { - const val MAX: Float = Float.MAX_VALUE - - /** - * Constant for the minimum float value. - */ - const val MIN: Float = -1f * Float.MAX_VALUE - } + private val vMax = Vec3f(MIN, MIN, MIN) /** - * Initial value for the maximum vector. The maximum vector contain the highest (positive if normalized) values for x, y and z of the bounding box. + * Initial value for the minimum vector. The minimum vector contains the lowest (negative if normalized) values for x, y, and z of the bounding box. */ - private val vMax = Vector3f(MIN, MIN, MIN) - - /** - * Initial value for the minimum vector. The minimum vector contain the lowest (negative if normalized) values for x, y and z of the bounding box. - */ - private val vMin = Vector3f(MAX, MAX, MAX) + private val vMin = Vec3f(MAX, MAX, MAX) /** * Center of mass of the bounding box as x, y, z vector. */ - private val com = Vector3f(0f, 0f, 0f) + private val com = Vec3f(0f, 0f, 0f) /** * Scaling factor to norm. The scaling factor is the factor to scale the bounding box to the norm. 1 for no scaling. */ - private var scalingFactorToNorm = 1f + var scalingFactorToNorm = 1f + private set /** * Translation to norm. The translation is the vector to translate com of the bounding box to the origin. (0, 0, 0) for no translation. */ - private val translationToNorm = Vector3f(0F, 0F, 0F) + val translationToNorm = Vec3f(0f, 0f, 0f) + + companion object { + /** + * Constant for the maximum float value. + */ + const val MAX = Float.MAX_VALUE + + /** + * Constant for the minimum float value. + */ + const val MIN = -1f * Float.MAX_VALUE + } /** - * Empty constructor to initialize an empty bounding box The purpose is to iteratively add bounding boxes. + * Empty constructor to initialize an empty bounding box. The purpose is to iteratively add bounding boxes. */ constructor() @@ -63,7 +65,7 @@ open class MinimalBoundingBox { * * @param positions List of vertices. */ - constructor(positions: List) { + constructor(positions: List) { update(positions) } @@ -73,8 +75,8 @@ open class MinimalBoundingBox { * * @return List of vertices. */ - private fun toList(): List { - val vec = mutableListOf() + fun toList(): List { + val vec = mutableListOf() if (isValidBoundingBox()) { vec.add(vMax) vec.add(vMin) @@ -97,42 +99,34 @@ open class MinimalBoundingBox { } /** - * Returns the scaling factor to norm size. - * - * @return Scaling factor to norm size. - */ - fun getScalingFactorToNorm(): Float { - return scalingFactorToNorm - } - - /** - * Get translation to Origin. + * Checks if the bounding box is valid. A bounding box is valid if each component of the maximum vector is greater than the corresponding component of the minimum vector. * - * @return Translation to Origin. + * @return True if the bounding box is valid, false otherwise. */ - fun getTranslationToNorm(): Vector3f { - return translationToNorm + private fun isValidBoundingBox(): Boolean { + return vMax.x > vMin.x && vMax.y > vMin.y && vMax.z > vMin.z } /** * Helper method to add data to the bounding box and recalculate the bounding boxes values. */ private fun update(positions: FloatArray) { - val vectors = positions.asList().chunked(3) { (x, y, z) -> Vector3f(x, y, z) } + val vectors = mutableListOf() + for (i in positions.indices step 3) { + vectors.add(Vec3f(positions[i], positions[i + 1], positions[i + 2])) + } update(vectors) } /** * Helper method to add data to the bounding box and recalculate the bounding boxes values. Since the calculation of the bounding box is iterative, the calculation is split into several steps. The steps are: - *
    - *
  • 1. Update the center of mass.
    • - *
  • 2. Update the scaling factor to norm.
    • - *
  • 3. Update the translation to norm.
    • - *
- * These steps had to be exact in this sequence - */ - private fun update(vec: List) { - // Has to be exact this sequence + * 1. Update the center of mass. + * 2. Update the scaling factor to norm. + * 3. Update the translation to norm. + * + * These steps had to be exact in this sequence. + */ + private fun update(vec: List) { if (updateBounds(vec)) { updateCom() updateScalingFactorToNorm() @@ -141,19 +135,10 @@ open class MinimalBoundingBox { } /** - * Checks if the bounding box is valid. A bounding box is valid if each component of the maximum vector is greater than the corresponding component of the minimum vector. - * - * @return True if the bounding box is valid, false otherwise. - */ - private fun isValidBoundingBox(): Boolean { - return vMax.x > vMin.x && vMax.y > vMin.y && vMax.z > vMin.z - } - - /** - * Update the center of mass. The center of mass is the middle point of the bounding box. + * Update the center of mass. The center of mass is the midpoint of the bounding box. */ private fun updateCom() { - com.set(Vector3f((vMax.x + vMin.x) / 2f, (vMax.y + vMin.y) / 2f, (vMax.z + vMin.z) / 2f)) + com.set((vMax.x + vMin.x) / 2f, (vMax.y + vMin.y) / 2f, (vMax.z + vMin.z) / 2f) } /** @@ -167,35 +152,35 @@ open class MinimalBoundingBox { * Update the scaling factor to norm. The scaling factor is the factor to scale the longest vector in the bounding box to the norm. 1 for no scaling. */ private fun updateScalingFactorToNorm() { - var farthest = Vector3f(0F, 0F, 0F) + var farthest = Vec3f(0f, 0f, 0f) for (vec in toList()) { - val vector = Vector3f(vec).sub(com) + val vector = Vec3f(vec).subtract(com) if (vector.length() > farthest.length()) { farthest = vector } - scalingFactorToNorm = 1f / (farthest.length() * 2) } + scalingFactorToNorm = 1f / (farthest.length() * 2) } /** - * Update the bounding box with new vectors + * Update the bounding box with new vectors. * * @return True if the bounding box has changed, false otherwise. */ - private fun updateBounds(positions: List): Boolean { + private fun updateBounds(positions: List): Boolean { var changed = false for (vec in positions) { - changed = updateBounds(vec) || changed + changed = changed or updateBounds(vec) } return changed } /** - * Update the bounding box with a new vector + * Update the bounding box with a new vector. * * @return True if the bounding box has changed, false otherwise. */ - private fun updateBounds(vec: Vector3f): Boolean { + private fun updateBounds(vec: Vec3f): Boolean { var changed = false if (vec.x > vMax.x) { vMax.x = vec.x @@ -223,4 +208,11 @@ open class MinimalBoundingBox { } return changed } + + /** + * Release all resources. + */ + fun close() { + // Placeholder for resource release logic if needed + } } diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/util/entropyoptimizer/EntopyCalculationMethod.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/util/entropyoptimizer/EntopyCalculationMethod.kt new file mode 100644 index 000000000..6a3260d39 --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/util/entropyoptimizer/EntopyCalculationMethod.kt @@ -0,0 +1,22 @@ +package org.vitrivr.engine.core.model.mesh.texturemodel.util.entropyoptimizer + +/** + * Options for the Entropy calculation. + */ +enum class EntopyCalculationMethod { + /** + * The entropy is calculated relative to the total area of the all faces. + */ + RELATIVE_TO_TOTAL_AREA, + + /** + * The entropy is calculated relative to the projected area of the faces. + */ + RELATIVE_TO_PROJECTED_AREA, + + /** + * The entropy is calculated relative to the projected area of the faces. + * Additionally, the weight of y component of the faces is taken into account. + */ + RELATIVE_TO_TOTAL_AREA_WEIGHTED, +} diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/util/entropyoptimizer/EntropyOptimizerStrategy.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/util/entropyoptimizer/EntropyOptimizerStrategy.kt new file mode 100644 index 000000000..b5ae9bcb0 --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/util/entropyoptimizer/EntropyOptimizerStrategy.kt @@ -0,0 +1,16 @@ +package org.vitrivr.engine.core.model.mesh.texturemodel.util.entropyoptimizer + +/** + * The method used to calculate the entropy. + */ +enum class EntropyOptimizerStrategy { + /** + * The new view vector is chosen randomly. + */ + RANDOMIZED, + + /** + * The new view vector is chosen by the gradient of the entropy. + */ + NEIGHBORHOOD, +} diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/util/entropyoptimizer/OptimizerOptions.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/util/entropyoptimizer/OptimizerOptions.kt new file mode 100644 index 000000000..2ee879657 --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/util/entropyoptimizer/OptimizerOptions.kt @@ -0,0 +1,47 @@ +package org.vitrivr.engine.core.model.mesh.texturemodel.util.entropyoptimizer + +import org.vitrivr.engine.core.model.mesh.texturemodel.util.types.Vec3f + + +/** + * Options for the ModelEntropyOptimizer and entropy calculation. + */ +data class OptimizerOptions( + /** + * The factor the unit vector is multiplied with to zoom. + * + * + * > 1 zooms out, < 1 zooms in. + */ + var zoomOutFactor: Float = 1f, //(float) Math.sqrt(3.0); + + /** + * The method used to optimize the entropy. + */ + var optimizer: EntropyOptimizerStrategy = EntropyOptimizerStrategy.RANDOMIZED, + + /** + * The method used to calculate the entropy. + */ + var method: EntopyCalculationMethod = EntopyCalculationMethod.RELATIVE_TO_TOTAL_AREA, + + /** + * The maximum number of iterations the optimizer should perform. + */ + var iterations: Int = 1000, + + /** + * The initial view vector. + */ + var initialViewVector: Vec3f = Vec3f(0f, 0f, 1f), + + /** + * Weight for y normal vectors pointing up. + */ + var yPosWeight: Float = 1f, + + /** + * Weight for y normal vectors pointing down. + */ + var yNegWeight: Float = 1f +) diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/util/types/Matrix4f.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/util/types/Matrix4f.kt new file mode 100644 index 000000000..2fac6b1aa --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/util/types/Matrix4f.kt @@ -0,0 +1,205 @@ +package org.vitrivr.engine.core.model.mesh.texturemodel.util.types + +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.nio.FloatBuffer + +/** + * Represents a 4x4 matrix, commonly used in 3D graphics for transformations including translation, scaling, and rotation. + */ +class Matrix4f { + + /** + * The matrix is stored as a single array of 16 floating-point elements in column-major order. + */ + var m: FloatArray // 4x4 matrix stored in a single array of 16 elements + + /** + * Constructor - initializes the matrix as an identity matrix. + */ + init { + m = FloatArray(16) + identity() + } + + /** + * Sets the matrix to the identity matrix. + * The identity matrix is a special type of matrix that does not alter the vector it multiplies. + * The matrix has 1s on the diagonal and 0s elsewhere. + * + * @return This matrix after being set to the identity matrix. + */ + fun identity(): Matrix4f { + m[0] = 1f + m[1] = 0f + m[2] = 0f + m[3] = 0f + m[4] = 0f + m[5] = 1f + m[6] = 0f + m[7] = 0f + m[8] = 0f + m[9] = 0f + m[10] = 1f + m[11] = 0f + m[12] = 0f + m[13] = 0f + m[14] = 0f + m[15] = 1f + return this + } + + /** + * Multiplies this matrix by another matrix and stores the result in this matrix. + * + * @param other The matrix to multiply this matrix by. + * @return This matrix after being multiplied by the specified matrix. + */ + fun mul(other: Matrix4f): Matrix4f { + val result = FloatArray(16) + + for (row in 0..3) { + for (col in 0..3) { + result[row * 4 + col] = + m[row * 4 + 0] * other.m[0 * 4 + col] + m[row * 4 + 1] * other.m[1 * 4 + col] + m[row * 4 + 2] * other.m[2 * 4 + col] + m[row * 4 + 3] * other.m[3 * 4 + col] + } + } + + m = result + return this + } + + /** + * Multiplies this matrix by a scalar value. + * + * @param scalar The scalar value to multiply each element of this matrix by. + * @return This matrix after being multiplied by the scalar. + */ + fun mul(scalar: Float): Matrix4f { + for (i in 0..15) { + m[i] *= scalar + } + return this + } + + + /** + * Sets this matrix to a scaling matrix. + * + * @param x The scaling factor along the X axis. + * @param y The scaling factor along the Y axis. + * @param z The scaling factor along the Z axis. + * @return This matrix after being set to the scaling matrix. + */ + fun scaling(x: Float, y: Float, z: Float): Matrix4f { + identity() + m[0] = x + m[5] = y + m[10] = z + return this + } + + + /** + * Applies this matrix to a vector and returns the transformed vector. + * + * @param v The vector to be transformed. + * @return The transformed vector. + */ + fun transform(v: Vec3f): Vec3f { + val x = v.x * m[0] + v.y * m[4] + v.z * m[8] + m[12] + val y = v.x * m[1] + v.y * m[5] + v.z * m[9] + m[13] + val z = v.x * m[2] + v.y * m[6] + v.z * m[10] + m[14] + return Vec3f(x, y, z) + } + + /** + * Sets this matrix to a combined translation, rotation (using a quaternion), and scaling matrix. + * + * @param translation The translation vector. + * @param rotation The rotation represented as a quaternion. + * @param scale The scaling factor. + * @return This matrix after being set to the combined translation, rotation, and scaling matrix. + */ + fun translationRotateScale(translation: Vec3f, rotation: Quaternionf, scale: Float): Matrix4f { + val xx = rotation.x * rotation.x + val xy = rotation.x * rotation.y + val xz = rotation.x * rotation.z + val xw = rotation.x * rotation.w + val yy = rotation.y * rotation.y + val yz = rotation.y * rotation.z + val yw = rotation.y * rotation.w + val zz = rotation.z * rotation.z + val zw = rotation.z * rotation.w + + m[0] = scale * (1.0f - 2.0f * (yy + zz)) + m[1] = scale * (2.0f * (xy + zw)) + m[2] = scale * (2.0f * (xz - yw)) + m[3] = 0.0f + + m[4] = scale * (2.0f * (xy - zw)) + m[5] = scale * (1.0f - 2.0f * (xx + zz)) + m[6] = scale * (2.0f * (yz + xw)) + m[7] = 0.0f + + m[8] = scale * (2.0f * (xz + yw)) + m[9] = scale * (2.0f * (yz - xw)) + m[10] = scale * (1.0f - 2.0f * (xx + yy)) + m[11] = 0.0f + + m[12] = translation.x + m[13] = translation.y + m[14] = translation.z + m[15] = 1.0f + + return this + } + + /** + * Provides a string representation of the matrix. + * + * @return A string representation of the matrix in a readable format. + */ + override fun toString(): String { + return """ + [${m[0]}, ${m[4]}, ${m[8]}, ${m[12]}] + [${m[1]}, ${m[5]}, ${m[9]}, ${m[13]}] + [${m[2]}, ${m[6]}, ${m[10]}, ${m[14]}] + [${m[3]}, ${m[7]}, ${m[11]}, ${m[15]}] + """.trimIndent() + } + + /** + * Copies the matrix components into a FloatBuffer. + * + * @param buffer The FloatBuffer to copy the matrix data into. + * @return The FloatBuffer with the matrix data. + */ + operator fun get(buffer: FloatBuffer): FloatBuffer { + buffer.clear() // Clear the buffer before putting data + buffer.put(m) // Put the matrix data into the buffer + buffer.flip() // Flip the buffer to prepare it for reading + return buffer + } + + /** + * Creates a FloatBuffer from this matrix's data. + * + * @return A FloatBuffer containing the matrix data. + */ + fun getFloatBuffer(): FloatBuffer { + // Allocate a direct ByteBuffer with capacity for the float array + val byteBuffer = ByteBuffer.allocateDirect(m.size * 4).order(ByteOrder.nativeOrder()) + + // Convert the ByteBuffer into a FloatBuffer + val floatBuffer = byteBuffer.asFloatBuffer() + + // Put the float array into the FloatBuffer + floatBuffer.put(m) + + // Prepare the buffer for reading (flip it) + floatBuffer.flip() + + return floatBuffer + } +} diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/util/types/Quaterinionf.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/util/types/Quaterinionf.kt new file mode 100644 index 000000000..ae320ed5e --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/util/types/Quaterinionf.kt @@ -0,0 +1,173 @@ +package org.vitrivr.engine.core.model.mesh.texturemodel.util.types + +import kotlin.math.cos +import kotlin.math.sin +import kotlin.math.sqrt + +/** + * Represents a quaternion, which is a complex number used to represent rotations in 3D space. + */ +class Quaternionf { + var x: Float = 0f + var y: Float = 0f + var z: Float = 0f + var w: Float = 0f + + /** + * Default constructor - initializes the quaternion as an identity quaternion (no rotation). + */ + constructor() { + identity() + } + + /** + * Constructor with specified components. + * + * @param x The x-component of the quaternion. + * @param y The y-component of the quaternion. + * @param z The z-component of the quaternion. + * @param w The w-component of the quaternion. + */ + constructor(x: Float, y: Float, z: Float, w: Float) { + this.x = x + this.y = y + this.z = z + this.w = w + } + + /** + * Sets the quaternion to the identity quaternion. + * The identity quaternion represents no rotation. + * + * @return This quaternion after being set to the identity quaternion. + */ + fun identity(): Quaternionf { + this.x = 0.0f + this.y = 0.0f + this.z = 0.0f + this.w = 1.0f + return this + } + + /** + * Normalizes the quaternion to ensure it represents a valid rotation. + * + * @return This quaternion after being normalized. + */ + fun normalize(): Quaternionf { + val length = sqrt((x * x + y * y + z * z + w * w).toDouble()).toFloat() + if (length != 0.0f) { + val invLength = 1.0f / length + this.x *= invLength + this.y *= invLength + this.z *= invLength + this.w *= invLength + } + return this + } + + /** + * Multiplies this quaternion by another quaternion. + * + * @param other The quaternion to multiply by. + * @return This quaternion after the multiplication. + */ + fun mul(other: Quaternionf): Quaternionf { + val newX = this.w * other.x + (this.x * other.w) + (this.y * other.z) - this.z * other.y + val newY = this.w * other.y + (this.y * other.w) + (this.z * other.x) - this.x * other.z + val newZ = this.w * other.z + (this.z * other.w) + (this.x * other.y) - this.y * other.x + val newW = this.w * other.w - (this.x * other.x) - (this.y * other.y) - (this.z * other.z) + + this.x = newX + this.y = newY + this.z = newZ + this.w = newW + + return this + } + + /** + * Multiplies this quaternion by a scalar value. + * + * @param scalar The scalar value to multiply each component of this quaternion by. + * @return This quaternion after being multiplied by the scalar. + */ + fun mul(scalar: Float): Quaternionf { + this.x *= scalar + this.y *= scalar + this.z *= scalar + this.w *= scalar + return this + } + + /** + * Computes the conjugate of this quaternion. + * The conjugate is used to compute the inverse and is obtained by negating the x, y, and z components. + * + * @return The conjugate of this quaternion. + */ + fun conjugate(): Quaternionf { + this.x = -this.x + this.y = -this.y + this.z = -this.z + return this + } + + /** + * Applies this quaternion to transform a vector. + * This is useful for rotating vectors in 3D space. + * + * @param v The vector to be transformed. + * @return The transformed vector. + */ + fun transform(v: Vec3f): Vec3f { + val vecQuat = Quaternionf(v.x, v.y, v.z, 0.0f) + val resQuat: Quaternionf = this.mul(vecQuat).mul(this.conjugate()) + return Vec3f(resQuat.x, resQuat.y, resQuat.z) + } + + /** + * Sets the quaternion to represent a rotation around an axis in radians. + * + * @param axisX The x-component of the rotation axis. + * @param axisY The y-component of the rotation axis. + * @param axisZ The z-component of the rotation axis. + * @param angle The angle of rotation in radians. + * @return This quaternion after being set to represent the rotation. + */ + fun fromAxisAngleRad(axisX: Float, axisY: Float, axisZ: Float, angle: Float): Quaternionf { + val halfAngle = angle / 2.0f + val sinHalfAngle = sin(halfAngle) + x = axisX * sinHalfAngle + y = axisY * sinHalfAngle + z = axisZ * sinHalfAngle + w = cos(halfAngle) + return this + } + + /** + * Sets the quaternion to represent a rotation around an axis in radians. + * + * @param vector3f The axis of rotation as a vector. + * @param angle The angle of rotation in radians. + * @return This quaternion after being set to represent the rotation. + */ + fun fromAxisAngleRad(vector3f: Vec3f, angle: Float): Quaternionf { + val halfAngle = angle / 2.0f + val sinHalfAngle = sin(halfAngle) + x = vector3f.x * sinHalfAngle + y = vector3f.y * sinHalfAngle + z = vector3f.z * sinHalfAngle + w = cos(halfAngle) + return this + } + + /** + * Provides a string representation of the quaternion. + * + * @return A string representing the quaternion in the format "Quaternionf(x, y, z, w)". + */ + override fun toString(): String { + return "Quaternionf($x, $y, $z, $w)" + } +} diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/util/types/Vec3f.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/util/types/Vec3f.kt new file mode 100644 index 000000000..8aaccff66 --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/util/types/Vec3f.kt @@ -0,0 +1,215 @@ +package org.vitrivr.engine.core.model.mesh.texturemodel.util.types + +import kotlin.math.sqrt +import java.io.Serializable + +/** + * Represents a 3D vector with floating-point coordinates. + * This class provides basic vector operations such as addition, subtraction, scaling, and normalization. + */ +class Vec3f +@JvmOverloads constructor(var x: Float = 0.0f, var y: Float = 0.0f, var z: Float = 0.0f) : Serializable { + + /** + * Copy constructor. + * + * @param other The vector to copy from. + */ + constructor(other: Vec3f) : this(other.x, other.y, other.z) + + /** + * Adds this vector to another vector. + * + * @param other The vector to add. + * @return A new vector that is the result of the addition. + */ + fun add(other: Vec3f): Vec3f { + return Vec3f(this.x + other.x, this.y + other.y, this.z + other.z) + } + + /** + * Subtracts another vector from this vector. + * + * @param other The vector to subtract. + * @return A new vector that is the result of the subtraction. + */ + fun subtract(other: Vec3f): Vec3f { + return Vec3f(this.x - other.x, this.y - other.y, this.z - other.z) + } + + /** + * Scales this vector by a scalar value. + * + * @param scalar The scalar value to scale by. + * @return A new vector that is the result of the scaling. + */ + fun scale(scalar: Float): Vec3f { + return Vec3f(this.x * scalar, this.y * scalar, this.z * scalar) + } + + /** + * Performs element-wise multiplication with another vector. + * + * @param other The vector to multiply with. + * @return A new vector that is the result of the element-wise multiplication. + */ + fun mul(other: Vec3f): Vec3f { + return Vec3f(this.x * other.x, this.y * other.y, this.z * other.z) + } + + /** + * Multiplies this vector by a scalar value. + * + * @param scalar The scalar value to multiply by. + * @return This vector after being multiplied by the scalar. + */ + fun mul(scalar: Float): Vec3f { + x *= scalar + y *= scalar + z *= scalar + return this + } + + /** + * Divides this vector by a scalar value. + * + * @param scalar The scalar value to divide by. + * @return A new vector that is the result of the division. + * @throws IllegalArgumentException if the scalar is zero. + */ + fun div(scalar: Float): Vec3f { + require(scalar != 0f) { "Cannot divide by zero." } + return Vec3f(this.x / scalar, this.y / scalar, this.z / scalar) + } + + /** + * Performs element-wise division with another vector. + * + * @param other The vector to divide by. + * @return A new vector that is the result of the element-wise division. + */ + fun div(other: Vec3f): Vec3f { + return Vec3f(this.x / other.x, this.y / other.y, this.z / other.z) + } + + /** + * Computes the cross product of this vector with another vector and stores the result in the given destination vector. + * + * @param v The vector to compute the cross product with. + * @param dest The destination vector to store the result. + * @return The destination vector with the cross product result. + */ + fun cross(v: Vec3f, dest: Vec3f): Vec3f { + val crossX: Float = this.y * v.z - this.z * v.y + val crossY: Float = this.z * v.x - this.x * v.z + val crossZ: Float = this.x * v.y - this.y * v.x + + dest.x = crossX + dest.y = crossY + dest.z = crossZ + + return dest + } + + /** + * Computes the magnitude (length) of the vector. + * + * @return The magnitude of the vector. + */ + fun length(): Float { + return sqrt((x * x + y * y + z * z).toDouble()).toFloat() + } + + /** + * Normalizes the vector to make it unit length. + * + * @return A new vector that is the normalized version of this vector. + */ + fun normalize(): Vec3f { + val length = length() + if (length != 0f) { + return Vec3f(this.x / length, this.y / length, this.z / length) + } + return Vec3f(0f, 0f, 0f) + } + + /** + * Sets the x, y, and z components of this vector to the given values. + * + * @param x The new x component. + * @param y The new y component. + * @param z The new z component. + * @return This vector after setting the new values. + */ + fun set(x: Float, y: Float, z: Float): Vec3f { + this.x = x + this.y = y + this.z = z + return this + } + + /** + * Sets the x, y, and z components of this vector to the values of another vector. + * + * @param other The vector to copy values from. + * @return This vector after setting the new values. + */ + fun set(other: Vec3f): Vec3f { + return set(other.x, other.y, other.z) + } + + /** + * Sets all components of this vector to zero. + * + * @return A new vector with all components set to zero. + */ + fun zero(): Vec3f { + return Vec3f(0f, 0f, 0f) + } + + /** + * Provides a string representation of the vector. + * + * @return A string representing the vector in the format "Vec3f(x, y, z)". + */ + override fun toString(): String { + return "Vec3f($x, $y, $z)" + } + + /** + * Compares this vector to another object for equality. + * + * @param obj The object to compare to. + * @return True if the object is a Vec3f with the same x, y, and z components, false otherwise. + */ + override fun equals(obj: Any?): Boolean { + if (this === obj) return true + if (obj == null || javaClass != obj.javaClass) return false + val vector = obj as Vec3f + return java.lang.Float.compare(vector.x, x) == 0 && java.lang.Float.compare( + vector.y, y + ) == 0 && java.lang.Float.compare(vector.z, z) == 0 + } + + /** + * Computes a hash code for this vector. + * + * @return A hash code value for the vector. + */ + override fun hashCode(): Int { + return java.lang.Float.hashCode(x) xor java.lang.Float.hashCode(y) xor java.lang.Float.hashCode(z) + } + + /** + * Computes the squared distance between this vector and another vector. + * + * @param v The vector to compute the distance to. + * @return The squared distance between this vector and the other vector. + */ + fun distanceSquared(v: Vec3f): Float { + val dx = this.x - v.x + val dy = this.y - v.y + val dz = this.z - v.z + return dx * dx + dy * dy + dz * dz + } +} diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/util/types/Vec4f.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/util/types/Vec4f.kt new file mode 100644 index 000000000..726c42d05 --- /dev/null +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/model/mesh/texturemodel/util/types/Vec4f.kt @@ -0,0 +1,149 @@ +package org.vitrivr.engine.core.model.mesh.texturemodel.util.types + +import java.io.Serializable + +/** + * Represents a 4D vector with floating-point coordinates. + * This class provides various operations for manipulating and interacting with 4D vectors. + * + * @property x The x component of the vector. + * @property y The y component of the vector. + * @property z The z component of the vector. + * @property w The w component of the vector. + */ +class Vec4f( + var x: Float = 0.0f, var y: Float = 0.0f, var z: Float = 0.0f, var w: Float = 0.0f +) : Serializable { + + /** + * Sets the components of this vector to the specified values. + * + * @param x The new x component. + * @param y The new y component. + * @param z The new z component. + * @param w The new w component. + * @return This vector with updated components. + */ + fun set(x: Float, y: Float, z: Float, w: Float): Vec4f { + this.x = x + this.y = y + this.z = z + this.w = w + return this + } + + /** + * Adds another vector to this vector. + * + * @param other The vector to add. + * @return This vector after addition. + */ + fun add(other: Vec4f): Vec4f { + x += other.x + y += other.y + z += other.z + w += other.w + return this + } + + /** + * Subtracts another vector from this vector. + * + * @param other The vector to subtract. + * @return This vector after subtraction. + */ + fun sub(other: Vec4f): Vec4f { + x -= other.x + y -= other.y + z -= other.z + w -= other.w + return this + } + + /** + * Multiplies this vector by a scalar. + * + * @param scalar The scalar value to multiply by. + * @return This vector after scaling. + */ + fun mul(scalar: Float): Vec4f { + x *= scalar + y *= scalar + z *= scalar + w *= scalar + return this + } + + /** + * Divides this vector by a scalar. + * + * @param scalar The scalar value to divide by. + * @return This vector after division. + * @throws ArithmeticException if the scalar is zero. + */ + fun div(scalar: Float): Vec4f { + val invScalar = 1.0f / scalar + x *= invScalar + y *= invScalar + z *= invScalar + w *= invScalar + return this + } + + /** + * Computes the length (magnitude) of the vector. + * + * @return The length of the vector. + */ + fun length(): Float { + return kotlin.math.sqrt(x * x + y * y + z * z + w * w) + } + + /** + * Normalizes the vector to make its length equal to 1. + * + * @return This vector after normalization. + */ + fun normalize(): Vec4f { + val len = length() + if (len != 0.0f) { + val invLen = 1.0f / len + x *= invLen + y *= invLen + z *= invLen + w *= invLen + } + return this + } + + /** + * Provides a string representation of the vector. + * + * @return A string representation in the format "Vector4f(x=x, y=y, z=z, w=w)". + */ + override fun toString(): String { + return "Vector4f(x=$x, y=$y, z=$z, w=$w)" + } + + /** + * Creates a copy of this vector. + * + * @return A new vector that is a copy of this vector. + */ + fun copy(): Vec4f { + return Vec4f(x, y, z, w) + } + + /** + * Checks if this vector is approximately equal to another vector. + * + * @param other The vector to compare with. + * @param epsilon The tolerance for floating-point comparison. + * @return True if the vectors are approximately equal, false otherwise. + */ + fun equals(other: Vec4f, epsilon: Float = 0.000001f): Boolean { + return kotlin.math.abs(x - other.x) < epsilon && kotlin.math.abs(y - other.y) < epsilon && kotlin.math.abs(z - other.z) < epsilon && kotlin.math.abs( + w - other.w + ) < epsilon + } +} diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/operators/transform/shape/CombineOperator.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/operators/transform/shape/CombineOperator.kt index 40f06ea33..35da7800f 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/operators/transform/shape/CombineOperator.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/operators/transform/shape/CombineOperator.kt @@ -34,8 +34,8 @@ class CombineOperator(override val inputs: List>) : p.toFlow(this).collect { it -> var send: T? = null mutex.lock() - buffer.compute(it.id) { _, v -> if (v == null) it to 1 else v.first to v.second + 1 } - if (buffer[it.id]?.second == require) { + val entry = buffer.compute(it.id) { _, v -> if (v == null) it to 1 else v.first to v.second + 1 } + if (entry?.second == require) { buffer.remove(it.id)?.first?.let { send = it } } mutex.unlock() diff --git a/vitrivr-engine-index/build.gradle b/vitrivr-engine-index/build.gradle index 9efc97a40..0a2000746 100644 --- a/vitrivr-engine-index/build.gradle +++ b/vitrivr-engine-index/build.gradle @@ -7,6 +7,7 @@ plugins { dependencies { /* vitrivr core dependency. */ api project(':vitrivr-engine-core') + api project(':vitrivr-engine-module-m3d') /** Java CV (used for video decoding). */ implementation group: 'org.bytedeco', name: 'javacv-platform', version: version_javacv diff --git a/vitrivr-engine-module-m3d/build.gradle b/vitrivr-engine-module-m3d/build.gradle index babdb025f..2ea0b4b3f 100644 --- a/vitrivr-engine-module-m3d/build.gradle +++ b/vitrivr-engine-module-m3d/build.gradle @@ -4,8 +4,6 @@ plugins { id 'signing' } -project.ext.lwjglVersion = "3.3.3" - switch (org.gradle.internal.os.OperatingSystem.current()) { case org.gradle.internal.os.OperatingSystem.LINUX: def osArch = System.getProperty("os.arch") @@ -33,50 +31,25 @@ dependencies { /** JOML dependencies for 3D mesh support. */ api group: 'org.joml', name: 'joml', version: version_joml - /* JOGL dependency for 3D rendering support. */ - api "org.jogamp.gluegen:gluegen-rt:${version_jogl}" - api "org.jogamp.jogl:jogl-all:${version_jogl}" - api "org.jogamp.jogl:jogl-all:${version_jogl}:natives-windows-amd64" - api "org.jogamp.jogl:jogl-all:${version_jogl}:natives-windows-i586" - api "org.jogamp.jogl:jogl-all:${version_jogl}:natives-linux-amd64" - api "org.jogamp.jogl:jogl-all:${version_jogl}:natives-linux-armv6" - api "org.jogamp.jogl:jogl-all:${version_jogl}:natives-linux-armv6hf" - api "org.jogamp.jogl:jogl-all:${version_jogl}:natives-linux-i586" - api "org.jogamp.jogl:jogl-all:${version_jogl}:natives-macosx-universal" - api "org.jogamp.gluegen:gluegen-rt:${version_jogl}:natives-windows-amd64" - api "org.jogamp.gluegen:gluegen-rt:${version_jogl}:natives-windows-i586" - api "org.jogamp.gluegen:gluegen-rt:${version_jogl}:natives-linux-amd64" - api "org.jogamp.gluegen:gluegen-rt:${version_jogl}:natives-linux-armv6" - api "org.jogamp.gluegen:gluegen-rt:${version_jogl}:natives-linux-armv6hf" - api "org.jogamp.gluegen:gluegen-rt:${version_jogl}:natives-linux-i586" - api "org.jogamp.gluegen:gluegen-rt:${version_jogl}:natives-macosx-universal" - /** LWJGL. Minimal OpenGl Configuration from customizer https://www.lwjgl.org/customize **/ - implementation platform("org.lwjgl:lwjgl-bom:$lwjglVersion") - - implementation "org.lwjgl:lwjgl" - implementation "org.lwjgl:lwjgl-assimp" - implementation "org.lwjgl:lwjgl-bgfx" - implementation "org.lwjgl:lwjgl-glfw" - implementation "org.lwjgl:lwjgl-nanovg" - implementation "org.lwjgl:lwjgl-nuklear" - implementation "org.lwjgl:lwjgl-openal" - implementation "org.lwjgl:lwjgl-opengl" - implementation "org.lwjgl:lwjgl-par" - implementation "org.lwjgl:lwjgl-stb" - implementation "org.lwjgl:lwjgl-vulkan" - runtimeOnly "org.lwjgl:lwjgl::$lwjglNatives" - runtimeOnly "org.lwjgl:lwjgl-assimp::$lwjglNatives" - runtimeOnly "org.lwjgl:lwjgl-bgfx::$lwjglNatives" - runtimeOnly "org.lwjgl:lwjgl-glfw::$lwjglNatives" - runtimeOnly "org.lwjgl:lwjgl-nanovg::$lwjglNatives" - runtimeOnly "org.lwjgl:lwjgl-nuklear::$lwjglNatives" - runtimeOnly "org.lwjgl:lwjgl-openal::$lwjglNatives" - runtimeOnly "org.lwjgl:lwjgl-opengl::$lwjglNatives" - runtimeOnly "org.lwjgl:lwjgl-par::$lwjglNatives" - runtimeOnly "org.lwjgl:lwjgl-stb::$lwjglNatives" - if (lwjglNatives == "natives-macos" || lwjglNatives == "natives-macos-arm64") runtimeOnly "org.lwjgl:lwjgl-vulkan::$lwjglNatives" + api group: "org.lwjgl", name: "lwjgl", version: version_lwjgl + api group: "org.lwjgl", name: "lwjgl-assimp", version: version_lwjgl + api group: "org.lwjgl", name: "lwjgl-glfw", version: version_lwjgl + api group: "org.lwjgl", name: "lwjgl-opengl", version: version_lwjgl + api group: "org.lwjgl", name: "lwjgl-stb", version: version_lwjgl + runtimeOnly group: "org.lwjgl", name: "lwjgl", version: version_lwjgl, classifier: project.ext.lwjglNatives + runtimeOnly group: "org.lwjgl", name: "lwjgl-assimp", version: version_lwjgl, classifier: project.ext.lwjglNatives + runtimeOnly group: "org.lwjgl", name: "lwjgl-bgfx", version: version_lwjgl, classifier: project.ext.lwjglNatives + runtimeOnly group: "org.lwjgl", name: "lwjgl-glfw", version: version_lwjgl, classifier: project.ext.lwjglNatives + runtimeOnly group: "org.lwjgl", name: "lwjgl-nanovg", version: version_lwjgl, classifier: project.ext.lwjglNatives + runtimeOnly group: "org.lwjgl", name: "lwjgl-nuklear", version: version_lwjgl, classifier: project.ext.lwjglNatives + runtimeOnly group: "org.lwjgl", name: "lwjgl-opengl", version: version_lwjgl, classifier: project.ext.lwjglNatives + runtimeOnly group: "org.lwjgl", name: "lwjgl-par", version: version_lwjgl, classifier: project.ext.lwjglNatives + runtimeOnly group: "org.lwjgl", name: "lwjgl-stb", version: version_lwjgl, classifier: project.ext.lwjglNatives + if (lwjglNatives == "natives-macos" || lwjglNatives == "natives-macos-arm64") { + runtimeOnly group: "org.lwjgl", name: "lwjgl-vulkan", version: version_lwjgl, classifier: project.ext.lwjglNatives + } } diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/ModelHandler.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/ModelHandler.kt deleted file mode 100644 index db1fd7bb3..000000000 --- a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/ModelHandler.kt +++ /dev/null @@ -1,297 +0,0 @@ -package org.vitrivr.engine.model3d - - -import org.apache.logging.log4j.LogManager -import org.joml.Vector4f -import org.lwjgl.BufferUtils -import org.lwjgl.assimp.* -import org.lwjgl.assimp.Assimp.* -import org.lwjgl.system.MemoryStack -import org.vitrivr.engine.core.model.mesh.* -import java.io.File -import java.io.FileInputStream -import java.io.InputStream -import java.nio.ByteBuffer -import java.nio.IntBuffer -import java.util.* - -class ModelHandler { - - - companion object { - private val LOGGER = LogManager.getLogger() - } - - /** - * Loads a model from a file. Generates all the standard flags for Assimp. For more details see Assimp. - *
    - *
  • aiProcess_GenSmoothNormals: - * This is ignored if normals are already there at the time this flag - * is evaluated. Model importers try to load them from the source file, so - * they're usually already there. - * This flag may not be specified together with - * #aiProcess_GenNormals. There's a configuration option, - * #AI_CONFIG_PP_GSN_MAX_SMOOTHING_ANGLE which allows you to specify - * an angle maximum for the normal smoothing algorithm. Normals exceeding - * this limit are not smoothed, resulting in a 'hard' seam between two faces. - * Using a decent angle here (e.g. 80 degrees) results in very good visual - * appearance. - *
    • - *
  • aiProcess_JoinIdenticalVertices:
    • - *
  • aiProcess_Triangulate By default the imported mesh data might contain faces with more than 3 - * indices. For rendering you'll usually want all faces to be triangles. - * This post processing step splits up faces with more than 3 indices into - * triangles. Line and point primitives are *not* modified! If you want - * 'triangles only' with no other kinds of primitives, try the following - * solution: - *
      - *
    • Specify both #aiProcess_Triangulate and #aiProcess_SortByPType
      • - * Ignore all point and line meshes when you process assimp's output - *
    - *
    • - *
  • aiProcess_FixInf acingNormals: - * This step tries to determine which meshes have normal vectors that are facing inwards and inverts them. - * The algorithm is simple but effective: the bounding box of all vertices + their normals is compared against - * the volume of the bounding box of all vertices without their normals. This works well for most objects, problems might occur with - * planar surfaces. However, the step tries to filter such cases. - * The step inverts all in-facing normals. Generally it is recommended to enable this step, although the result is not always correct. - *
    • - *
  • aiProcess_CalcTangentSpace: - * Calculates the tangents and bi tangents for the imported meshes - * Does nothing if a mesh does not have normals. - * You might want this post processing step to be executed if you plan to use tangent space calculations such as normal mapping applied to the meshes. - * There's an importer property, AI_CONFIG_PP_CT_MAX_SMOOTHING_ANGLE, which allows you to specify a maximum smoothing angle for the algorithm. - * However, usually you'll want to leave it at the default value. - *
    • - *
  • aiProcess_LimitBoneWeights: - * Limits the number of bones simultaneously affecting a single vertex to a maximum value. - * If any vertex is affected by more than the maximum number of bones, - * the least important vertex weights are removed and the remaining vertex weights are normalized so that the weights still sum up to 1. - * The default bone weight limit is 4 (defined as AI_LBW_MAX_WEIGHTS in config.h), - * but you can use the AI_CONFIG_PP_LBW_MAX_WEIGHTS importer property to supply your own limit to the post processing step. - * If you intend to perform the skinning in hardware, this post processing step might be of interest to you. - *
    • - *
  • aiProcess_PreTransformVertices: - * Removes the node graph and pre-transforms all vertices with the local transformation matrices of their nodes. - * If the resulting scene can be reduced to a single mesh, with a single material, no lights, and no cameras, - * then the output scene will contain only a root node (with no children) that references the single mesh. - * Otherwise, the output scene will be reduced to a root node with a single level of child nodes, each one referencing one mesh, - * and each mesh referencing one material - * In either case, for rendering, you can simply render all meshes in order - you don't need to pay attention to local transformations and the node hierarchy. - * Animations are removed during this step - * This step is intended for applications without a scenegraph. - * The step CAN cause some problems: if e.g. a mesh of the asset contains normals and another, using the same material index, - * does not, they will be brought together, but the first mesh's part of the normal list is zeroed. However, these artifacts are rare. - *
    • - *
- * - * @param modelId The ID of the model. - * @param modelPath InputStream id the model. - * @return Model object. - */ - fun loadModel(modelId: String, inputStream: InputStream): Model3D { - LOGGER.trace("Try loading model {} from InputStream", modelId) - - val aiScene = loadAIScene(modelId, inputStream) - val materialList: MutableList = ArrayList() - val defaultMaterial = Material(mutableListOf()) // Define a default material - val aiMeshes = aiScene.mMeshes() - val numMeshes = aiScene.mNumMeshes() - for (ic in 0 until numMeshes) { - val aiMesh = AIMesh.create(aiMeshes?.get(ic) ?: continue) - val mesh = processMesh(aiMesh) - val materialIdx = aiMesh.mMaterialIndex() - val material = if (materialIdx >= 0 && materialIdx < materialList.size) { - materialList[materialIdx] - } else { - defaultMaterial - } - material.meshes.add(mesh) - } - - if (defaultMaterial.meshes.isNotEmpty()) { - materialList.add(defaultMaterial) - } - - aiReleaseImport(aiScene) - - return Model3D(modelId, materials = materialList) - } - - /** - * Loads a model from a file path. - * - * @param modelId The ID of the model. - * @param modelPath Path to the model file. - * @return Model object. - */ - fun loadModel(modelId: String, modelPath: String): Model3D { - val file = File(modelPath) - val inputStream = FileInputStream(file) - return loadModel(modelId, inputStream) - } - - private fun processIndices(aiMesh: AIMesh): IntArray { - LOGGER.trace("Start processing indices") - val indices: MutableList = ArrayList() - val numFaces = aiMesh.mNumFaces() - val aiFaces = aiMesh.mFaces() - for (ic in 0 until numFaces) { - val aiFace = aiFaces[ic] - val buffer: IntBuffer = aiFace.mIndices() - while (buffer.remaining() > 0) { - indices.add(buffer.get()) - } - } - LOGGER.trace("End processing indices") - return indices.toIntArray() - } - - private fun processMaterial(aiMaterial: AIMaterial, modelDir: String): Material { - LOGGER.trace("Start processing material") - val material = Material() - MemoryStack.stackPush().use { stack -> - val color = AIColor4D.create() - - //** Diffuse color if no texture is present - val result = aiGetMaterialColor(aiMaterial, AI_MATKEY_COLOR_DIFFUSE, aiTextureType_NONE, 0, color) - if (result == aiReturn_SUCCESS) { - material.diffuseColor = Vector4f(color.r(), color.g(), color.b(), color.a()) - } - - //** Try load texture - val aiTexturePath = AIString.calloc(stack) - aiGetMaterialTexture( - aiMaterial, aiTextureType_DIFFUSE, 0, aiTexturePath, null as IntBuffer?, null, null, null, null, null - ) - val texturePath = aiTexturePath.dataString() - //TODO: Warning - if (texturePath.isNotEmpty()) { - material.texture = Texture(File(modelDir + File.separator + File(texturePath).toPath())) - material.diffuseColor = Material.DEFAULT_COLOR - } - - return material - } - } - - private fun processMesh(aiMesh: AIMesh): Mesh { - LOGGER.trace("Start processing mesh") - val vertices = processVertices(aiMesh) - val normals = processNormals(aiMesh) - var textCoords = processTextCoords(aiMesh) - val indices = processIndices(aiMesh) - - // Texture coordinates may not have been populated. We need at least the empty slots - if (textCoords.isEmpty()) { - val numElements = vertices.size / 3 * 2 - textCoords = FloatArray(numElements) - } - LOGGER.trace("End processing mesh") - return Mesh.of(vertices, normals,textCoords, indices) - } - - private fun processNormals(aiMesh: AIMesh): FloatArray? { - LOGGER.trace("Start processing Normals") - val buffer = aiMesh.mNormals() ?: return null - val data = FloatArray(buffer.remaining() * 3) - var pos = 0 - while (buffer.remaining() > 0) { - val normal = buffer.get() - data[pos++] = normal.x() - data[pos++] = normal.y() - data[pos++] = normal.z() - } - return data - } - - private fun processTextCoords(aiMesh: AIMesh): FloatArray { - LOGGER.trace("Start processing Coordinates") - val buffer = aiMesh.mTextureCoords(0) ?: return floatArrayOf() - val data = FloatArray(buffer.remaining() * 2) - var pos = 0 - while (buffer.remaining() > 0) { - val textCoord = buffer.get() - data[pos++] = textCoord.x() - data[pos++] = 1 - textCoord.y() - } - return data - } - - private fun processVertices(aiMesh: AIMesh): FloatArray { - LOGGER.trace("Start processing Vertices") - val buffer = aiMesh.mVertices() - val data = FloatArray(buffer.remaining() * 3) - var pos = 0 - while (buffer.remaining() > 0) { - val textCoord = buffer.get() - data[pos++] = textCoord.x() - data[pos++] = textCoord.y() - data[pos++] = textCoord.z() - } - - return data - } - - /** - * Loads a AIScene from a file. Generates all the standard flags for Assimp. For more details see Assimp. - */ - private fun loadAIScene(modelId: String, inputStream: InputStream): AIScene { - LOGGER.trace("Try loading model {} from InputStream", modelId) - - val data = inputStream.readBytes() - val buffer = BufferUtils.createByteBuffer(data.size) - buffer.put(data) - buffer.flip() - - val aiScene = aiImportFileFromMemory(buffer, aiProcess_JoinIdenticalVertices or aiProcess_GlobalScale or aiProcess_FixInfacingNormals or aiProcess_Triangulate or aiProcess_CalcTangentSpace or aiProcess_LimitBoneWeights or aiProcess_PreTransformVertices, null as ByteBuffer?) - - if (aiScene == null) { - throw RuntimeException("Error loading model from InputStream") - } - - return aiScene - } - - - - /** - * Export the model to a file. (gltf or obj) - * - * @param model The model to export. - * @param format The format in which the file will be saved (e.g., "gltf" or "obj"). - * @param outputPath The path where the model will be saved. - */ - fun export(aiScene: AIScene, format: String, outputPath: String) { - if ((format.lowercase(Locale.getDefault()) != "obj") and (format.lowercase(Locale.getDefault()) != "gltf")) { - throw RuntimeException("Error exporting scene to $outputPath. Format not supported: $format") - } - - val exportFlags = aiProcess_Triangulate or aiProcess_FlipUVs - - println("Exporting scene to $outputPath") - - if (format.lowercase(Locale.getDefault()) == "obj") { - // Export the scene to OBJ format - val result = aiExportScene(aiScene, format, outputPath, exportFlags) - - if (result == aiReturn_SUCCESS) { - println("Scene exported successfully to $outputPath") - } else { - println("Error exporting scene to $outputPath. Result code: $result") - } - } else if (format.lowercase(Locale.getDefault()) == "gltf") { - // Export the scene to glTF format - // Set additional export flags for glTF 2.0 - val gltfExportFlags = exportFlags or aiProcessPreset_TargetRealtime_MaxQuality - - val result = aiExportScene(aiScene, "gltf2", outputPath, gltfExportFlags) - - if (result == aiReturn_SUCCESS) { - println("Scene exported successfully to $outputPath") - } else { - println("Error exporting scene to $outputPath. Result code: $result") - } - } - } -} \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/ModelLoader.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/ModelLoader.kt new file mode 100644 index 000000000..2d3a8dea6 --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/ModelLoader.kt @@ -0,0 +1,494 @@ +package org.vitrivr.engine.model3d + +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import org.lwjgl.BufferUtils +import org.lwjgl.assimp.* +import org.lwjgl.assimp.Assimp.* +import org.lwjgl.system.MemoryStack +import org.lwjgl.system.MemoryUtil +import org.vitrivr.engine.core.model.mesh.texturemodel.Material +import org.vitrivr.engine.core.model.mesh.texturemodel.Mesh +import org.vitrivr.engine.core.model.mesh.texturemodel.Model3d +import org.vitrivr.engine.core.model.mesh.texturemodel.Texture +import org.vitrivr.engine.core.model.mesh.texturemodel.util.types.Vec4f +import java.awt.image.BufferedImage +import java.io.ByteArrayInputStream +import java.io.File +import java.io.IOException +import java.io.InputStream +import java.nio.ByteBuffer +import java.nio.IntBuffer +import javax.imageio.ImageIO + +class ModelLoader { + private val LOGGER: Logger = LogManager.getLogger() + + /** + * Loads a model from a file. Generates all the standard flags for Assimp. For more details see + * [Assimp](https://javadoc.lwjgl.org/org/lwjgl/assimp/Assimp.html). + * * **aiProcess_GenSmoothNormals:** This is ignored if normals are already there at the time this + * flag is evaluated. Model importers try to load them from the source file, so they're usually + * already there. This flag may not be specified together with + * + * #aiProcess_GenNormals. There's a configuration option, + * #AI_CONFIG_PP_GSN_MAX_SMOOTHING_ANGLE which allows you to specify an angle maximum for + * the normal smoothing algorithm. Normals exceeding this limit are not smoothed, resulting in a + * 'hard' seam between two faces. Using a decent angle here (e.g. 80 degrees) results in very good + * visual appearance. + * * **aiProcess_JoinIdenticalVertices:** + * * **aiProcess_Triangulate** By default the imported mesh data might contain faces with more + * than 3 indices. For rendering you'll usually want all faces to be triangles. This post + * processing step splits up faces with more than 3 indices into triangles. Line and point + * primitives are *not* modified! If you want 'triangles only' with no other kinds of + * primitives, try the following solution: + * * Specify both #aiProcess_Triangulate and #aiProcess_SortByPType Ignore all point and line + * meshes when you process assimp's output + * * **aiProcess_FixInf acingNormals:** This step tries to determine which meshes have normal + * vectors that are facing inwards and inverts them. The algorithm is simple but effective: the + * bounding box of all vertices + their normals is compared against the volume of the bounding + * box of all vertices without their normals. This works well for most objects, problems might + * occur with planar surfaces. However, the step tries to filter such cases. The step inverts + * all in-facing normals. Generally it is recommended to enable this step, although the result + * is not always correct. + * * **aiProcess_CalcTangentSpace:** Calculates the tangents and bi tangents for the imported + * meshes Does nothing if a mesh does not have normals. You might want this post processing step + * to be executed if you plan to use tangent space calculations such as normal mapping applied + * to the meshes. There's an importer property, AI_CONFIG_PP_CT_MAX_SMOOTHING_ANGLE, which + * allows you to specify a maximum smoothing angle for the algorithm. However, usually you'll + * want to leave it at the default value. + * * **aiProcess_LimitBoneWeights:** Limits the number of bones simultaneously affecting a single + * vertex to a maximum value. If any vertex is affected by more than the maximum number of + * bones, the least important vertex weights are removed and the remaining vertex weights are + * normalized so that the weights still sum up to 1. The default bone weight limit is 4 (defined + * as AI_LBW_MAX_WEIGHTS in config.h), but you can use the AI_CONFIG_PP_LBW_MAX_WEIGHTS importer + * property to supply your own limit to the post processing step. If you intend to perform the + * skinning in hardware, this post processing step might be of interest to you. + * * **aiProcess_PreTransformVertices:** Removes the node graph and pre-transforms all vertices + * with the local transformation matrices of their nodes. If the resulting scene can be reduced + * to a single mesh, with a single material, no lights, and no cameras, then the output scene + * will contain only a root node (with no children) that references the single mesh. Otherwise, + * the output scene will be reduced to a root node with a single level of child nodes, each one + * referencing one mesh, and each mesh referencing one material In either case, for rendering, + * you can simply render all meshes in order - you don't need to pay attention to local + * transformations and the node hierarchy. Animations are removed during this step This step is + * intended for applications without a scenegraph. The step CAN cause some problems: if e.g. a + * mesh of the asset contains normals and another, using the same material index, does not, they + * will be brought together, but the first mesh's part of the normal list is zeroed. However, + * these artifacts are rare. + * + * @param modelId The ID of the model. + * @param modelPath Path to the model file. + * @return Model object. + */ + fun loadModel(modelId: String?, inputStream: InputStream): Model3d? { + val model = + loadModel( + modelId, + inputStream, + aiProcess_JoinIdenticalVertices or + aiProcess_GlobalScale or + aiProcess_FixInfacingNormals or + aiProcess_Triangulate or + aiProcess_CalcTangentSpace or + aiProcess_LimitBoneWeights or + aiProcess_PreTransformVertices) + LOGGER.trace("Try return Model 2") + return model + } + + // Keep function if we want to extend at a later point to load models again from path to get external textures + fun loadModelPath(modelId: String?, path: String): Model3d? { + val model = + loadModel( + modelId, + path, + aiProcess_JoinIdenticalVertices or + aiProcess_GlobalScale or + aiProcess_FixInfacingNormals or + aiProcess_Triangulate or + aiProcess_CalcTangentSpace or + aiProcess_LimitBoneWeights or + aiProcess_PreTransformVertices) + LOGGER.trace("Try return Model 2") + return model + } + + /** + * Loads a model from a file. 1. Loads the model file to an aiScene. 2. Process all Materials. 3. + * Process all Meshes. 3.1 Process all Vertices. 3.2 Process all Normals. 3.3 Process all + * Textures. 3.4 Process all Indices. + * + * @param modelId Arbitrary unique ID of the model. + * @param inputStream InputStream where the model file is. + * @param flags Flags for the model loading process. + * @return Model object. + */ + private fun loadModel(modelId: String?, inputStream: InputStream, flags: Int): Model3d? { + LOGGER.trace("Try loading file {}.", modelId) + + val aiScene = modelId?.let { loadAIScene(it, inputStream) } + + /*val file = File(modelPath) + if (!file.exists()) { + throw RuntimeException("Model path does not exist [$modelPath]") + } + val modelDir = file.parent + + LOGGER.trace("Loading aiScene")*/ + + // DO NOT USE AUTOCLOSEABLE TRY CATCH FOR AI-SCENE!!! THIS WILL CAUSE A FATAL ERROR ON NTH (199) + // ITERATION! + // RAPHAEL WALTENSPUEL 2023-01-20 + /*val aiScene = + Assimp.aiImportFile(modelPath, flags) + ?: throw RuntimeException("Error loading model [modelPath: $modelPath]") +*/ + val numMaterials = aiScene?.mNumMaterials() + val materialList: MutableList = ArrayList() + + // TODO: Warning + for (ic in 0 until numMaterials!!) { + val aiMaterial = AIMaterial.create(aiScene.mMaterials()!![ic]) + LOGGER.trace("Try processing material {}", ic) + materialList.add(processMaterial(aiMaterial, aiScene)) + } + + val numMeshes = aiScene.mNumMeshes() + val aiMeshes = aiScene.mMeshes() + val defaultMaterial = Material() + for (ic in 0 until numMeshes) { + LOGGER.trace("Try create AI Mesh {}", ic) + // TODO: Warning + val aiMesh = AIMesh.create(aiMeshes!![ic]) + val mesh = processMesh(aiMesh) + LOGGER.trace("Try get Material idx") + val materialIdx = aiMesh.mMaterialIndex() + val material = + if (materialIdx >= 0 && materialIdx < materialList.size) { + materialList[materialIdx] + } else { + defaultMaterial + } + LOGGER.trace("Try add Material to Mesh") + material.addMesh(mesh) + } + + if (defaultMaterial.materialMeshes.isNotEmpty()) { + LOGGER.trace("Try add default Material") + materialList.add(defaultMaterial) + } + + LOGGER.trace("Try instantiate Model") + aiReleaseImport(aiScene) + + val model3d = modelId?.let { Model3d(it, materialList) } + LOGGER.trace("Try return Model") + return model3d + } + + + /** + * Loads a model from a file. 1. Loads the model file to an aiScene. 2. Process all Materials. 3. + * Process all Meshes. 3.1 Process all Vertices. 3.2 Process all Normals. 3.3 Process all + * Textures. 3.4 Process all Indices. + * + * @param modelId Arbitrary unique ID of the model. + * @param modelPath String to the model file. + * @param flags Flags for the model loading process. + * @return Model object. + */ + fun loadModel(modelId: String?, modelPath: String, flags: Int): Model3d? { + LOGGER.trace("Try loading file {}.", modelId) + + + val file = File(modelPath) + if (!file.exists()) { + throw RuntimeException("Model path does not exist [$modelPath]") + } + + LOGGER.trace("Loading aiScene") + + // DO NOT USE AUTOCLOSEABLE TRY CATCH FOR AI-SCENE!!! THIS WILL CAUSE A FATAL ERROR ON NTH (199) + // ITERATION! + // RAPHAEL WALTENSPUEL 2023-01-20 + val aiScene = + aiImportFile(modelPath, flags) + ?: throw RuntimeException("Error loading model [modelPath: $modelPath]") + + val numMaterials = aiScene.mNumMaterials() + val materialList: MutableList = ArrayList() + + // TODO: Warning + for (ic in 0 until numMaterials!!) { + val aiMaterial = AIMaterial.create(aiScene.mMaterials()!![ic]) + LOGGER.trace("Try processing material {}", ic) + materialList.add(processMaterial(aiMaterial, aiScene)) + } + + val numMeshes = aiScene.mNumMeshes() + val aiMeshes = aiScene.mMeshes() + val defaultMaterial = Material() + for (ic in 0 until numMeshes) { + LOGGER.trace("Try create AI Mesh {}", ic) + // TODO: Warning + val aiMesh = AIMesh.create(aiMeshes!![ic]) + val mesh = processMesh(aiMesh) + LOGGER.trace("Try get Material idx") + val materialIdx = aiMesh.mMaterialIndex() + var material = + if (materialIdx >= 0 && materialIdx < materialList.size) { + materialList[materialIdx] + } else { + defaultMaterial + } + LOGGER.trace("Try add Material to Mesh") + material.addMesh(mesh) + } + + if (defaultMaterial.materialMeshes.isNotEmpty()) { + LOGGER.trace("Try add default Material") + materialList.add(defaultMaterial) + } + + LOGGER.trace("Try instantiate Model") + aiReleaseImport(aiScene) + + val model3d = modelId?.let { Model3d(it, materialList) } + LOGGER.trace("Try return Model") + return model3d + } + + + + /** + * Loads a AIScene from a file. Generates all the standard flags for Assimp. For more details see Assimp. + */ + private fun loadAIScene(modelId: String, inputStream: InputStream): AIScene { + LOGGER.trace("Try loading model {} from InputStream", modelId) + + val data = inputStream.readBytes() + val buffer = BufferUtils.createByteBuffer(data.size) + buffer.put(data) + buffer.flip() + + val aiScene = aiImportFileFromMemory(buffer, aiProcess_JoinIdenticalVertices or aiProcess_GlobalScale or aiProcess_FixInfacingNormals or aiProcess_Triangulate or aiProcess_CalcTangentSpace or aiProcess_LimitBoneWeights or aiProcess_PreTransformVertices, null as ByteBuffer?) + ?: throw RuntimeException("Error loading model from InputStream") + + return aiScene + } + + /** + * Convert indices from aiMesh to int array. + * + * @param aiMesh aiMesh to process. + * @return flattened int array of indices. + */ + private fun processIndices(aiMesh: AIMesh): IntArray { + LOGGER.trace("Start processing indices") + val indices: MutableList = ArrayList() + val numFaces = aiMesh.mNumFaces() + val aiFaces = aiMesh.mFaces() + for (ic in 0 until numFaces) { + val aiFace = aiFaces[ic] + val buffer = aiFace.mIndices() + while (buffer.remaining() > 0) { + indices.add(buffer.get()) + } + } + LOGGER.trace("End processing indices") + return indices.stream().mapToInt { obj: Int -> obj }.toArray() + } + + /** + * Convert an AIMaterial to a Material. Loads the diffuse color and texture. + * + * @param aiMaterial aiMaterial to process. + * @param modelDir Path to the model file. + * @param aiScene AIScene to process. + * @return flattened float array of vertices. + */ + private fun processMaterial( + aiMaterial: AIMaterial, + aiScene: AIScene + ): Material { + LOGGER.trace("Start processing material") + val material = Material() + MemoryStack.stackPush().use { stack -> + val color = AIColor4D.create() + val result = + aiGetMaterialColor( + aiMaterial, AI_MATKEY_COLOR_DIFFUSE, aiTextureType_NONE, 0, color) + if (result == aiReturn_SUCCESS) { + material.materialDiffuseColor = Vec4f(color.r(), color.g(), color.b(), color.a()) + } + + val aiTexturePath = AIString.calloc(stack) + aiGetMaterialTexture( + aiMaterial, + aiTextureType_DIFFUSE, + 0, + aiTexturePath, + null as IntBuffer?, + null, + null, + null, + null, + null) + val texturePath = aiTexturePath.dataString() + LOGGER.debug("Texture path: {}", texturePath) + if (texturePath != null && texturePath.isNotEmpty()) { + if (texturePath[0] == '*') { + val textureIndex = texturePath.substring(1).toInt() + LOGGER.debug("Embedded texture index: {}", textureIndex) + val image = loadEmbeddedTexture(aiScene, textureIndex) + if (image != null) { + material.materialTexture = Texture(image) + material.materialDiffuseColor = Material.DEFAULT_COLOR + } else { + LOGGER.error("Failed to load embedded texture at index {}", textureIndex) + } + } else { + /*LOGGER.debug("Texture file path: {}", modelDir + File.separator + texturePath) + material.materialTexture = Texture(modelDir + File.separator + File(texturePath).toPath())*/ + material.materialDiffuseColor = Material.DEFAULT_COLOR + } + } else { + //LOGGER.warn("No texture path found for material") + } + return material + } + } + + /** + * Convert aiMesh to a Mesh. Loads the vertices, normals, texture coordinates and indices. + * Instantiates a new Mesh object. + * + * @param aiScene AIScene to process. + * @return flattened float array of normals. + */ + private fun loadEmbeddedTexture(aiScene: AIScene, textureIndex: Int): BufferedImage? { + try { + val aiTextures = aiScene.mTextures() + if (aiTextures == null || textureIndex >= aiTextures.limit()) { + LOGGER.error("No textures or invalid texture index {}", textureIndex) + return null + } + + val aiTexture = AITexture.create(aiTextures[textureIndex]) + if (aiTexture == null) { + LOGGER.error("Failed to retrieve texture at index {}", textureIndex) + return null + } + + val address = aiTexture.pcData().address0() + val buffer = MemoryUtil.memByteBuffer(address, aiTexture.mWidth()) + + // Read the data into a byte array + val texData = ByteArray(buffer.remaining()) + buffer[texData] + + // Create BufferedImage from decoded data + val bis = ByteArrayInputStream(texData) + val image = ImageIO.read(bis) + bis.close() + + if (image != null) { + LOGGER.debug("Successfully loaded embedded texture") + } else { + LOGGER.error("Failed to decode embedded texture to BufferedImage") + } + + return image + } catch (e: IOException) { + LOGGER.error("Error loading embedded texture", e) + return null + } catch (e: Exception) { + LOGGER.error("Error processing embedded texture", e) + return null + } + } + + /** + * Convert aiMesh to a Mesh. Loads the vertices, normals, texture coordinates and indices. + * Instantiates a new Mesh object. + * + * @param aiMesh aiMesh to process. + * @return flattened float array of normals. + */ + private fun processMesh(aiMesh: AIMesh): Mesh { + LOGGER.trace("Start processing mesh") + val vertices = processVertices(aiMesh) + val normals = processNormals(aiMesh) + var textCoords = processTextCoords(aiMesh) + val indices = processIndices(aiMesh) + + // Texture coordinates may not have been populated. We need at least the empty slots + if (textCoords.isEmpty()) { + val numElements = (vertices.size / 3) * 2 + textCoords = FloatArray(numElements) + } + LOGGER.trace("End processing mesh") + return Mesh(vertices, normals, textCoords, indices) + } + + /** + * Convert normals from aiMesh to float array. + * + * @param aiMesh aiMesh to process. + * @return flattened float array of normals. + */ + private fun processNormals(aiMesh: AIMesh): FloatArray? { + LOGGER.trace("Start processing Normals") + val buffer = aiMesh.mNormals() ?: return null + val data = FloatArray(buffer.remaining() * 3) + var pos = 0 + while (buffer.remaining() > 0) { + val normal = buffer.get() + data[pos++] = normal.x() + data[pos++] = normal.y() + data[pos++] = normal.z() + } + return data + } + + /** + * Convert texture coordinates from aiMesh to float array. + * + * @param aiMesh aiMesh to process. + * @return flattened float array of texture coordinates. + */ + private fun processTextCoords(aiMesh: AIMesh): FloatArray { + LOGGER.trace("Start processing Coordinates") + val buffer = aiMesh.mTextureCoords(0) ?: return floatArrayOf() + val data = FloatArray(buffer.remaining() * 2) + var pos = 0 + while (buffer.remaining() > 0) { + val textCoord = buffer.get() + data[pos++] = textCoord.x() + data[pos++] = 1 - textCoord.y() + } + return data + } + + /** + * Convert vertices from aiMesh to float array. + * + * @param aiMesh aiMesh to process. + * @return flattened float array of vertices. + */ + private fun processVertices(aiMesh: AIMesh): FloatArray { + LOGGER.trace("Start processing Vertices") + val buffer = aiMesh.mVertices() + val data = FloatArray(buffer.remaining() * 3) + var pos = 0 + while (buffer.remaining() > 0) { + val textCoord = buffer.get() + data[pos++] = textCoord.x() + data[pos++] = textCoord.y() + data[pos++] = textCoord.z() + } + + return data + } +} diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/ModelPreviewExporter.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/ModelPreviewExporter.kt new file mode 100644 index 000000000..237ebca52 --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/ModelPreviewExporter.kt @@ -0,0 +1,320 @@ +package org.vitrivr.engine.model3d + +import io.github.oshai.kotlinlogging.KLogger +import io.github.oshai.kotlinlogging.KotlinLogging +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.onCompletion +import kotlinx.coroutines.flow.onEach +import org.joml.Vector3f +import org.vitrivr.engine.core.context.IndexContext +import org.vitrivr.engine.core.model.mesh.texturemodel.Model3d +import org.vitrivr.engine.core.model.mesh.texturemodel.util.entropyoptimizer.EntopyCalculationMethod +import org.vitrivr.engine.core.model.mesh.texturemodel.util.entropyoptimizer.EntropyOptimizerStrategy +import org.vitrivr.engine.core.model.mesh.texturemodel.util.entropyoptimizer.OptimizerOptions +import org.vitrivr.engine.core.model.retrievable.Retrievable +import org.vitrivr.engine.core.model.retrievable.attributes.SourceAttribute +import org.vitrivr.engine.core.model.mesh.texturemodel.util.types.Vec3f +import org.vitrivr.engine.core.operators.Operator +import org.vitrivr.engine.core.operators.general.Exporter +import org.vitrivr.engine.core.operators.general.ExporterFactory +import org.vitrivr.engine.core.source.MediaType +import org.vitrivr.engine.core.source.file.MimeType +import org.vitrivr.engine.model3d.lwjglrender.render.RenderOptions +import org.vitrivr.engine.model3d.lwjglrender.window.WindowOptions +import org.vitrivr.engine.model3d.lwjglrender.util.texturemodel.entroopyoptimizer.ModelEntropyOptimizer +import org.vitrivr.engine.model3d.renderer.ExternalRenderer +import java.awt.image.BufferedImage +import java.io.ByteArrayOutputStream +import java.util.* +import javax.imageio.* +import javax.imageio.metadata.IIOMetadata +import javax.imageio.metadata.IIOMetadataNode +import javax.imageio.stream.MemoryCacheImageOutputStream +import kotlin.math.cos +import kotlin.math.sin +import kotlin.math.sqrt + +private val logger: KLogger = KotlinLogging.logger {} + +/** + * An [Exporter] that generates a preview of a 3d model. + * + * @author Rahel Arnold + * @version 2.1.0 + */ +class ModelPreviewExporter : ExporterFactory { + companion object { + val SUPPORTED = setOf(MimeType.GLTF) + + /** Set of supported output formats. */ + val OUTPUT_FORMAT = setOf("gif", "jpg") + + /** + * Renders a preview of the given model as a JPEG image. + * + * @param model The [Model3d] to render. + * @param renderer The [ExternalRenderer] to use for rendering. + * @param distance The distance of the camera from the model. + * @return [BufferedImage] + */ + fun renderPreviewJPEG(model: Model3d, renderer: ExternalRenderer, distance: Float = 1.0f): BufferedImage { + if (model.modelMaterials.isNotEmpty()) { + // Set options for the renderer. + val windowOptions = + object : WindowOptions(400, 400) { + init { + hideWindow = true + } + } + val renderOptions = RenderOptions( + showTextures = true + ) + + + // Set options for the entropy optimizer. + val opts = OptimizerOptions( + iterations = 100, + initialViewVector = Vec3f(0f, 0f, 1f), + method = EntopyCalculationMethod.RELATIVE_TO_TOTAL_AREA_WEIGHTED, + optimizer = EntropyOptimizerStrategy.RANDOMIZED, + yNegWeight = 0.7f, + yPosWeight = 0.8f + ) + + // Define camera positions + val cameraPositions = LinkedList() + cameraPositions.add( + Vector3f( + (Math.random() - 0.5).toFloat() * 2f, + (Math.random() - 0.5).toFloat() * 2f, + (Math.random() - 0.5).toFloat() * 2f + ) + .normalize() + .mul(distance) + ) + cameraPositions.add(Vector3f(0f, 0f, 1f).normalize().mul(distance)) + cameraPositions.add(Vector3f(-1f, 1f, 1f).normalize().mul(distance)) + cameraPositions.add(ModelEntropyOptimizer.getViewVectorWithMaximizedEntropy(model, opts)) + + /* Render the model. */ + val images = renderer.render(model, cameraPositions, windowOptions, renderOptions) + require(images.size == 4) { "Expected 4 images, but got ${images.size}." } + + /* Combine images into a single image. */ + val combinedImage = BufferedImage(800, 800, BufferedImage.TYPE_INT_RGB) + val g = combinedImage.graphics + g.drawImage(images[0], 0, 0, null) // Top-left + g.drawImage(images[1], images[0].width, 0, null) // Top-right + g.drawImage(images[2], 0, images[0].height, null) // Bottom-left + g.drawImage(images[3], images[0].width, images[0].height, null) + g.dispose() + + return combinedImage + } + throw IllegalArgumentException("Model has no materials.") + } + + /** + * + */ + fun createFramesForGif(model: Model3d, renderer: ExternalRenderer, views: Int, distance: Float = 1.0f): List { + if (model.modelMaterials.isNotEmpty()) { + // Set options for the renderer. + val windowOptions = + object : WindowOptions(400, 400) { + init { + hideWindow = true + } + } + val renderOptions = RenderOptions( + showTextures = true + ) + + // Define camera positions depending on the number of views. + val camera = generateCameraPositions(views, distance) + val images = renderer.render(model, camera, windowOptions, renderOptions) + + assert(images.size == views) + return images + } + throw IllegalArgumentException("Model has no materials.") + } + + + /** + * Generates camera positions for a given number of views. + */ + fun generateCameraPositions(numViews: Int, distance: Float): List { + val cameraPositions = LinkedList() + val goldenAngle = Math.PI * (3 - sqrt(5.0)) // Golden angle in radians + + for (i in 0 until numViews) { + val y = 1 - (i / (numViews - 1.0)) * 2 // y goes from 1 to -1 + val radius = sqrt(1 - y * y) // radius at y + + val theta = goldenAngle * i // angle increment + + val x = (cos(theta) * radius).toFloat() + val z = (sin(theta) * radius).toFloat() + cameraPositions.add(Vector3f(x, y.toFloat(), z).normalize().mul(distance)) + } + + return cameraPositions + } + } + + /** + * Creates a new [Exporter] instance from this [ModelPreviewExporter]. + * + * @param name The name of the [Exporter] + * @param input The [Operator] to acting as an input. + * @param context The [IndexContext] to use. + */ + override fun newExporter( + name: String, + input: Operator, + context: IndexContext + ): Exporter { + val maxSideResolution = context[name, "maxSideResolution"]?.toIntOrNull() ?: 800 + val mimeType = + context[name, "mimeType"]?.let { + try { + MimeType.valueOf(it.uppercase()) + } catch (e: java.lang.IllegalArgumentException) { + null + } + } ?: MimeType.GLTF + val distance = context[name, "distance"]?.toFloatOrNull() ?: 1f + val format = context[name, "format"] ?: "gif" + val views = context[name, "views"]?.toIntOrNull() ?: 30 + + logger.debug { + "Creating new ModelPreviewExporter with maxSideResolution=$maxSideResolution and mimeType=$mimeType" + } + return Instance(input, context, maxSideResolution, mimeType, distance, format, views) + } + + /** The [Exporter] generated by this [ModelPreviewExporter]. */ + private class Instance( + override val input: Operator, + private val context: IndexContext, + private val maxResolution: Int, + mimeType: MimeType, + private val distance: Float, + private val format: String, + private val views: Int + ) : Exporter { + init { + require(mimeType in SUPPORTED) { "ModelPreviewExporter only supports models of format GLTF." } + require(format in OUTPUT_FORMAT) { "ModelPreviewExporter only supports exporting a gif of jpg." } + } + + override fun toFlow(scope: CoroutineScope): Flow { + val renderer = ExternalRenderer() + return this.input.toFlow(scope).onEach { retrievable -> + val source = + retrievable.filteredAttribute(SourceAttribute::class.java)?.source ?: return@onEach + if (source.type == MediaType.MESH) { + val resolvable = this.context.resolver.resolve(retrievable.id) + + val model = retrievable.content[0].content as Model3d + if (resolvable != null) { + logger.debug { + "Generating preview for ${retrievable.id} with ${retrievable.type} and resolution $maxResolution. Storing it with ${resolvable::class.simpleName}." + } + + source.newInputStream().use { input -> + if (format == "jpg") { + val preview: BufferedImage = renderPreviewJPEG(model, renderer, this.distance) + resolvable.openOutputStream().use { output -> + ImageIO.write(preview, "jpg", output) + } + } else { // format == "gif" + val frames = createFramesForGif(model, renderer, this.views, this.distance) + val gif = createGif(frames, 50) + resolvable.openOutputStream().use { output -> output.write(gif!!.toByteArray()) } + } + } + } + } + }.onCompletion { + renderer.close() + } + } + + /** + * + */ + fun createGif(frames: List, delayTimeMs: Int): ByteArrayOutputStream? { + if (frames.isEmpty()) { + println("No frames to write to GIF.") + return null + } + + try { + val byteArrayOutputStream = ByteArrayOutputStream() + val outputImageStream = MemoryCacheImageOutputStream(byteArrayOutputStream) + val gifWriter: ImageWriter? = ImageIO.getImageWritersBySuffix("gif").next() + + if (gifWriter != null) { + gifWriter.output = outputImageStream + gifWriter.prepareWriteSequence(null) + + val param: ImageWriteParam = gifWriter.defaultWriteParam + param.compressionMode = ImageWriteParam.MODE_EXPLICIT + + val delayTime = (delayTimeMs).toString() + + for (frame in frames) { + val image = IIOImage(frame, null, getMetadata(gifWriter, delayTime)) + gifWriter.writeToSequence(image, param) + } + + gifWriter.endWriteSequence() + outputImageStream.close() + gifWriter.dispose() + + // println("GIF created successfully.") + return byteArrayOutputStream + } else { + println("Failed to create GIF writer.") + } + } catch (e: Exception) { + e.printStackTrace() + } + return null + } + + private fun getMetadata(gifWriter: ImageWriter, delayTime: String): IIOMetadata { + val imageType = ImageTypeSpecifier.createFromBufferedImageType(BufferedImage.TYPE_INT_RGB) + val metadata = gifWriter.getDefaultImageMetadata(imageType, null) + val metaFormatName = metadata.nativeMetadataFormatName + + val root = IIOMetadataNode(metaFormatName) + val graphicsControlExtensionNode = IIOMetadataNode("GraphicControlExtension") + + graphicsControlExtensionNode.setAttribute("disposalMethod", "none") + graphicsControlExtensionNode.setAttribute("userInputFlag", "FALSE") + graphicsControlExtensionNode.setAttribute("transparentColorFlag", "FALSE") + graphicsControlExtensionNode.setAttribute("delayTime", delayTime) + graphicsControlExtensionNode.setAttribute("transparentColorIndex", "0") + + root.appendChild(graphicsControlExtensionNode) + + val appExtensionsNode = IIOMetadataNode("ApplicationExtensions") + val appExtensionNode = IIOMetadataNode("ApplicationExtension") + + appExtensionNode.setAttribute("applicationID", "NETSCAPE") + appExtensionNode.setAttribute("authenticationCode", "2.0") + appExtensionNode.userObject = byteArrayOf(0x1, 0x0, 0x0) + + appExtensionsNode.appendChild(appExtensionNode) + root.appendChild(appExtensionsNode) + + metadata.mergeTree(metaFormatName, root) + + return metadata + } + } +} diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/decoder/MeshDecoder.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/decoder/MeshDecoder.kt index 3f8df78d4..14c63656a 100644 --- a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/decoder/MeshDecoder.kt +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/decoder/MeshDecoder.kt @@ -14,7 +14,7 @@ import org.vitrivr.engine.core.operators.ingest.DecoderFactory import org.vitrivr.engine.core.operators.ingest.Enumerator import org.vitrivr.engine.core.source.MediaType import org.vitrivr.engine.core.source.Source -import org.vitrivr.engine.model3d.ModelHandler +import org.vitrivr.engine.model3d.ModelLoader import java.io.IOException /** @@ -56,11 +56,11 @@ class MeshDecoder : DecoderFactory { logger.info { "Decoding source ${source.name} (${source.sourceId})" } try { - val handler = ModelHandler() + val handler = ModelLoader() val model = source.newInputStream().use { handler.loadModel(source.sourceId.toString(), it) // Pass InputStream directly } - val modelContent = this.context.contentFactory.newMeshContent(model) + val modelContent = this.context.contentFactory.newMeshContent(model!!) sourceRetrievable.addContent(modelContent) sourceRetrievable.addAttribute(ContentAuthorAttribute(modelContent.id, this.name)) sourceRetrievable diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/features/sphericalharmonics/SphericalHarmonics.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/features/sphericalharmonics/SphericalHarmonics.kt index 6ece552b3..2ccbb603c 100644 --- a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/features/sphericalharmonics/SphericalHarmonics.kt +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/features/sphericalharmonics/SphericalHarmonics.kt @@ -9,7 +9,7 @@ import org.vitrivr.engine.core.context.QueryContext import org.vitrivr.engine.core.model.content.element.ContentElement import org.vitrivr.engine.core.model.content.element.Model3DContent import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor -import org.vitrivr.engine.core.model.mesh.Mesh +import org.vitrivr.engine.core.model.mesh.texturemodel.Mesh import org.vitrivr.engine.core.model.metamodel.Analyser import org.vitrivr.engine.core.model.metamodel.Schema import org.vitrivr.engine.core.model.query.Query @@ -37,7 +37,7 @@ private val logger: KLogger = KotlinLogging.logger {} * @author Ralph Gasser * @version 1.2.0 */ -class SphericalHarmonics: Analyser { +class SphericalHarmonics : Analyser { companion object { /** Name of the grid_size parameter; determines size of voxel grid for rasterization. */ const val GRID_SIZE_PARAMETER_NAME = "grid_size" @@ -87,7 +87,7 @@ class SphericalHarmonics: Analyser { * @param minL The minimum L parameter to obtain [SphericalHarmonics] function value for. * @param maxL The maximum L parameter to obtain [SphericalHarmonics] function value for. */ - fun analyse(mesh: Mesh, gridSize: Int, cap: Int, minL: Int, maxL: Int): FloatVectorDescriptor { + fun analyse(mesh: org.vitrivr.engine.core.model.mesh.texturemodel.Mesh, gridSize: Int, cap: Int, minL: Int, maxL: Int): FloatVectorDescriptor { val voxelizer = Voxelizer(2.0f / gridSize) val increment = 0.1 /* Increment of the angles during calculation of the descriptors. */ val R: Int = gridSize / 2 @@ -192,7 +192,7 @@ class SphericalHarmonics: Analyser { val cap = field.parameters[CAP_PARAMETER_NAME]?.toIntOrNull() ?: CAP_PARAMETER_DEFAULT val minL = field.parameters[MINL_PARAMETER_NAME]?.toIntOrNull() ?: MINL_PARAMETER_DEFAULT val maxL = field.parameters[MAXL_PARAMETER_NAME]?.toIntOrNull() ?: MAXL_PARAMETER_DEFAULT - val descriptors = content.map { analyse(it.content.getMaterials().first().meshes.first(), gridSize, cap, minL, maxL) } + val descriptors = content.map { analyse(it.content.getMaterials().first().materialMeshes.first(), gridSize, cap, minL, maxL) } /* Return retriever. */ return this.newRetrieverForDescriptors(field, descriptors, context) diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/features/sphericalharmonics/SphericalHarmonicsExtractor.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/features/sphericalharmonics/SphericalHarmonicsExtractor.kt index 62416b0be..fd445cd71 100644 --- a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/features/sphericalharmonics/SphericalHarmonicsExtractor.kt +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/features/sphericalharmonics/SphericalHarmonicsExtractor.kt @@ -53,6 +53,6 @@ class SphericalHarmonicsExtractor( */ override fun extract(retrievable: Retrievable): List { val content = retrievable.content.filterIsInstance() - return content.flatMap { c -> c.content.getMaterials().flatMap { mat -> mat.meshes.map { mesh -> SphericalHarmonics.analyse(mesh, this.gridSize, this.minL, this.maxL, this.cap).copy(field = this.field) } } } + return content.flatMap { c -> c.content.getMaterials().flatMap { mat -> mat.materialMeshes.map { mesh -> SphericalHarmonics.analyse(mesh, this.gridSize, this.minL, this.maxL, this.cap).copy(field = this.field) } } } } } \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/engine/Engine.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/engine/Engine.kt new file mode 100644 index 000000000..25b4936d5 --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/engine/Engine.kt @@ -0,0 +1,176 @@ +package org.vitrivr.engine.model3d.lwjglrender.engine + +import org.vitrivr.engine.model3d.lwjglrender.glmodel.GLScene +import org.vitrivr.engine.model3d.lwjglrender.render.Render +import org.vitrivr.engine.model3d.lwjglrender.render.RenderOptions +import org.vitrivr.engine.model3d.lwjglrender.scene.Camera +import org.vitrivr.engine.model3d.lwjglrender.scene.Scene +import org.vitrivr.engine.model3d.lwjglrender.window.Window +import org.vitrivr.engine.model3d.lwjglrender.window.WindowOptions + +/** + * The engine is the main class of the rendering engine. It holds the window, the scene, and the + * render object. It provides a render loop for continuous rendering and a runOnce method to render + * a single frame rendering. + */ +class Engine(windowTitle: String, opts: WindowOptions, private val appLogic: EngineLogic) { + /** The window object. */ + private val window: Window = + Window(windowTitle, opts) { + resize() + null + } + + /** Indicates whether the engine is running in continuous rendering mode. */ + private var running: Boolean = true + + /** The render object. */ + private val render: Render = Render() + + /** The scene object. */ + private val scene: GLScene = GLScene(Scene(window.width, window.height)) + + /** The target frames per second. */ + private val targetFps: Int = opts.fps + + /** The target updates per second. (e.g. inputs rotation, etc.) */ + private val targetUps: Int = opts.ups + + init { + appLogic.init(window, scene, render) + } + + /** + * Sets the render options. Must be called before render is called. + * + * @param options The render options. + */ + fun setRenderOptions(options: RenderOptions) { + render.setOptions(options) + } + + /** + * Refreshes the engine. Is called when the engine is stopped and has to be ready to start again. + */ + fun refresh() { + appLogic.cleanup() + render.cleanup() + scene.cleanup() + } + + /** + * Releases all resources and terminates the engine. Is called when the engine is stopped and all + * resources have to be released. + */ + fun clear() { + appLogic.cleanup() + render.cleanup() + scene.cleanup() + window.cleanup() + } + + /** Starts the engine in continuous rendering mode. */ + fun start() { + running = true + run() + } + + /** Stops the continuous rendering mode. */ + fun stop() { + running = false + } + + /** Runs a single frame rendering. */ + fun runOnce() { + window.pollEvents() + appLogic.beforeRender(window, scene, render) + render.render(window, scene) + appLogic.afterRender(window, scene, render) + window.update() + } + + /** + * Run mode runs permanently until the engine is stopped. + * 1. Poll events + * 2. Input + * 3. Update + * 4. Render + * 5. Update window + */ + fun run() { + var initialTime = System.currentTimeMillis() + // maximum elapsed time between updates + val timeU = 1000.0f / targetUps + // maximum elapsed time between render calls + val timeR = if (targetFps > 0) 1000.0f / targetFps else 0f + var deltaUpdate = 0.0f + var deltaFps = 0.0f + + var updateTime = initialTime + + while (running && !window.windowShouldClose()) { + window.pollEvents() + + val now = System.currentTimeMillis() + + // relation between actual and elapsed time. 1 if equal. + deltaUpdate += (now - initialTime) / timeU + deltaFps += (now - initialTime) / timeR + + // If passed maximum elapsed time for render, process user input + if (targetFps <= 0 || deltaFps >= 1) { + appLogic.input(window, scene, now - initialTime) + } + + // If passed maximum elapsed time for update, update the scene + if (deltaUpdate >= 1) { + val diffTimeMillis = now - updateTime + appLogic.update(window, scene, diffTimeMillis) + updateTime = now + deltaUpdate-- + } + + // If passed maximum elapsed time for render, render the scene + if (targetFps <= 0 || deltaFps >= 1) { + appLogic.beforeRender(window, scene, render) + render.render(window, scene) + deltaFps-- + window.update() + appLogic.afterRender(window, scene, render) + } + } + refresh() + } + + /** Resizes the window. */ + fun resize() { + scene.resize(window.width, window.height) + } + + /** + * Returns the camera object. + * + * @return The camera object. + */ + fun getCamera(): Camera { + return scene.getCamera() + } + + /** + * Returns the window object. + * + * @return The window object. + */ + fun getWindow(): Window { + return window + } + + /** + * Returns the scene object. + * + * @return The scene object. + */ + fun getScene(): GLScene { + return scene + } +} diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/engine/EngineLogic.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/engine/EngineLogic.kt new file mode 100644 index 000000000..803f87409 --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/engine/EngineLogic.kt @@ -0,0 +1,48 @@ +package org.vitrivr.engine.model3d.lwjglrender.engine + +import org.vitrivr.engine.model3d.lwjglrender.glmodel.GLScene +import org.vitrivr.engine.model3d.lwjglrender.render.Render +import org.vitrivr.engine.model3d.lwjglrender.window.Window + +/** The EngineLogic provides methods to be called by the engine on certain states. */ +abstract class EngineLogic { + + /** + * Is called from the engine as first step during refresh and cleanup. @implSpec DO NOT CALL + * ENGINE LOGIC METHODS IN THIS METHOD DO NOT CALL THIS METHOD FROM EXTENDING CLASS + */ + internal abstract fun cleanup() + + /** + * Is called once at the initialization of the engine. @implSpec DO NOT CALL ENGINE LOGIC METHODS + * IN THIS METHOD DO NOT CALL THIS METHOD FROM EXTENDING CLASS + */ + internal abstract fun init(window: Window, scene: GLScene, render: Render) + + /** + * Is called from the engine before the render method. @implSpec DO NOT CALL ENGINE LOGIC METHODS + * IN THIS METHOD DO NOT CALL THIS METHOD FROM EXTENDING CLASS + */ + internal abstract fun beforeRender(window: Window, scene: GLScene, render: Render) + + /** + * Is called from the engine after the render method. @implSpec DO NOT CALL ENGINE LOGIC METHODS + * IN THIS METHOD DO NOT CALL THIS METHOD FROM EXTENDING CLASS + */ + internal abstract fun afterRender(window: Window, scene: GLScene, render: Render) + + /** + * This method is called every frame. This is only used in continuous rendering. The purpose is to + * do some input handling. Could be used to optimize view angles in a fast manner. @implSpec DO + * NOT CALL ENGINE LOGIC METHODS IN THIS METHOD DO NOT CALL THIS METHOD FROM EXTENDING CLASS + */ + internal abstract fun input(window: Window, scene: GLScene, diffTimeMillis: Long) + + /** + * After Engine run this method is called every frame. This is only used in continuous rendering. + * The purpose is to process some live output. Could be used to optimize view angles in a fast + * manner. @implSpec DO NOT CALL ENGINE LOGIC METHODS IN THIS METHOD DO NOT CALL THIS METHOD FROM + * EXTENDING CLASS + */ + internal abstract fun update(window: Window, scene: GLScene, diffTimeMillis: Long) +} diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/glmodel/GLMaterial.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/glmodel/GLMaterial.kt new file mode 100644 index 000000000..ecaf9bd58 --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/glmodel/GLMaterial.kt @@ -0,0 +1,44 @@ +package org.vitrivr.engine.model3d.lwjglrender.glmodel + +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import org.vitrivr.engine.core.model.mesh.texturemodel.Material +import org.vitrivr.engine.core.model.mesh.texturemodel.util.types.Vec4f + +/** + * The GLMaterial class is a wrapper for the [Material] class. + * * Material -> GLMaterial( Material ) + * + * The purpose is to bring the generic Material in an OpenGl context [Material] -> [GLMaterial] + */ +data class GLMaterial( + /** The material that is wrapped by this gl material. */ + val material: Material, + /** The contained meshes in gl context */ + val meshes: List = material.getMeshes().map { GLMesh(it!!) }, + /** The contained texture in gl context */ + val texture: GLTexture = GLTexture(material.materialTexture!!) +) { + /** + * Cleans up the gl material and calls all underlying cleanup methods. Removes only the references + * to wrapped generic meshes and texture. Hence, the material could be used by another extraction + * task this method does not close the generic meshes or texture. + */ + fun cleanup() { + meshes.forEach { it.cleanup() } + texture.cleanup() + LOGGER.trace("Cleaned-up GLMaterial") + } + + /** + * Returns the color from wrapped generic material. + * + * @return The color from wrapped generic material. (r,g,b,opacity) + */ + val diffuseColor: Vec4f + get() = material.materialDiffuseColor + + companion object { + private val LOGGER: Logger = LogManager.getLogger() + } +} diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/glmodel/GLMesh.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/glmodel/GLMesh.kt new file mode 100644 index 000000000..62570e846 --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/glmodel/GLMesh.kt @@ -0,0 +1,110 @@ +package org.vitrivr.engine.model3d.lwjglrender.glmodel + +import java.util.function.Consumer +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import org.lwjgl.opengl.GL30 +import org.lwjgl.system.MemoryStack +import org.lwjgl.system.MemoryUtil +import org.vitrivr.engine.core.model.mesh.texturemodel.Mesh + +/** + * The GLMesh class is a wrapper for the [Mesh] class. + * * Mesh -> GLMesh( Mesh ) + * + * The purpose is to bring the generic Mesh in an OpenGl context [Mesh] -> [GLMesh] + */ +class GLMesh( + /** The wrapped generic mesh in gl context */ + private val mesh: Mesh +) { + /** The list of *Vertex Buffer Object* (VBO) ids */ + private val vboIdList: MutableList = ArrayList() + /** + * Returns the *Vertex Array Object* (VAO) id. + * + * @return The *Vertex Array Object* (VAO) id. + */ + /** The *Vertex Array Object* (VAO) id */ + var vaoId: Int = 0 + + /** + * Creates a new GLMesh from a mesh. + * 1. Bind Vertex Array Object + * 1. Generate, allocate and initialize Vertex (Positions) Buffer + * 1. Generate, allocate and initialize Texture Coordinates Buffer + * 1. Generate, allocate and initialize Index Buffer + * 1. Unbind Vertex Array Object + * + * @param mesh The mesh that is wrapped by this gl mesh. + */ + init { + MemoryStack.stackPush().use { memoryStack -> + this.vaoId = GL30.glGenVertexArrays() + GL30.glBindVertexArray(this.vaoId) + + // Positions VBO + var vboId = GL30.glGenBuffers() + vboIdList.add(vboId) + val positionsBuffer = memoryStack.callocFloat(mesh.getPositions().size) + positionsBuffer.put(0, mesh.getPositions()) + GL30.glBindBuffer(GL30.GL_ARRAY_BUFFER, vboId) + GL30.glBufferData(GL30.GL_ARRAY_BUFFER, positionsBuffer, GL30.GL_STATIC_DRAW) + GL30.glEnableVertexAttribArray(0) + GL30.glVertexAttribPointer(0, 3, GL30.GL_FLOAT, false, 0, 0) + + // Textures VBO (Vertex Buffer Object) + vboId = GL30.glGenBuffers() + vboIdList.add(vboId) + val textureCoordinatesBuffer = MemoryUtil.memAllocFloat(mesh.getTextureCoords().size) + textureCoordinatesBuffer.put(0, mesh.getTextureCoords()) + GL30.glBindBuffer(GL30.GL_ARRAY_BUFFER, vboId) + GL30.glBufferData(GL30.GL_ARRAY_BUFFER, textureCoordinatesBuffer, GL30.GL_STATIC_DRAW) + GL30.glEnableVertexAttribArray(1) + GL30.glVertexAttribPointer(1, 2, GL30.GL_FLOAT, false, 0, 0) + + // Index VBO (Vertex Buffer Object) + vboId = GL30.glGenBuffers() + vboIdList.add(vboId) + val idxBuffer = memoryStack.callocInt(mesh.getIdx().size) + idxBuffer.put(0, mesh.getIdx()) + GL30.glBindBuffer(GL30.GL_ELEMENT_ARRAY_BUFFER, vboId) + GL30.glBufferData(GL30.GL_ELEMENT_ARRAY_BUFFER, idxBuffer, GL30.GL_STATIC_DRAW) + + GL30.glBindBuffer(GL30.GL_ARRAY_BUFFER, 0) + GL30.glBindVertexArray(0) + } + } + + /** + * Cleans up the gl mesh and calls all underlying cleanup methods. Removes only the references to + * VBOs and VAOs. Removes the *Vertex Array Object* (VAO) and all *Vertex Buffer Object* (VBO) + * ids. + */ + fun cleanup() { + vboIdList.forEach(Consumer { buffer: Int? -> GL30.glDeleteBuffers(buffer!!) }) + GL30.glDeleteVertexArrays(this.vaoId) + vboIdList.clear() + LOGGER.trace("Cleaned-up GLMesh") + } + + val numVertices: Int + /** + * Returns the number of vertices of the wrapped generic mesh. + * + * @return The number of vertices of the wrapped generic mesh. + */ + get() = mesh.getNumVertices() + + val id: String? + /** + * Returns the ID of the wrapped generic mesh. + * + * @return The ID of the wrapped generic mesh. + */ + get() = mesh.id + + companion object { + private val LOGGER: Logger = LogManager.getLogger() + } +} diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/glmodel/GLModel.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/glmodel/GLModel.kt new file mode 100644 index 000000000..b82f3affb --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/glmodel/GLModel.kt @@ -0,0 +1,70 @@ +package org.vitrivr.engine.model3d.lwjglrender.glmodel + +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import org.vitrivr.engine.core.model.mesh.texturemodel.Entity +import org.vitrivr.engine.core.model.mesh.texturemodel.IModel + +/** + * The GLModel class is a wrapper for the [IModel] class. + *
    + *
  • IModel -> GLModel( IModel )
    • + *
+ * + * The purpose is to bring the generic IModel into an OpenGL context + * [IModel] -> [GLModel] + */ +class GLModel(private val model: IModel) : IGLModel { + + /** + * The contained materials in GL context + */ + private val materials: MutableList = mutableListOf() + + init { + model.getMaterials().forEach { material -> + materials.add(GLMaterial(material)) + } + } + + /** + * {@inheritDoc} + */ + override fun getEntities(): List { + return model.getEntities() + } + + /** + * {@inheritDoc} + */ + override fun addEntity(entity: Entity) { + model.addEntity(entity) + } + + /** + * {@inheritDoc} + */ + override fun cleanup() { + materials.forEach { it.cleanup() } + materials.clear() + LOGGER.debug("GLModel cleaned up") + } + + /** + * {@inheritDoc} + */ + override fun getId(): String { + return model.getId() + } + + /** + * {@inheritDoc} + */ + override fun getMaterials(): List { + return materials + } + + companion object { + private val LOGGER: Logger = LogManager.getLogger(GLModel::class.java) + } +} diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/glmodel/GLScene.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/glmodel/GLScene.kt new file mode 100644 index 000000000..b33130d48 --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/glmodel/GLScene.kt @@ -0,0 +1,142 @@ +package org.vitrivr.engine.model3d.lwjglrender.glmodel + +import org.vitrivr.engine.model3d.lwjglrender.scene.Camera +import org.vitrivr.engine.model3d.lwjglrender.scene.Projection +import org.vitrivr.engine.model3d.lwjglrender.scene.Scene +import org.vitrivr.engine.core.model.mesh.texturemodel.Entity +import org.vitrivr.engine.core.model.mesh.texturemodel.IModel + +/** + * The GLScene class is the topmost class of the GL model hierarchy. + * The GL model hierarchy is used as a wrapper for the model hierarchy. + * Therefore, each GL class has a corresponding model class. + * The generic class has to be provided in the constructor. + *
    + *
  • Scene -> GLScene( Scene )
    • + *
  • Model -> GlModel( IModel )
    • + *
  • Material -> GLMaterial( Material )
    • + *
  • Mesh -> GLMesh( Mesh )
    • + *
  • Texture -> GLTexture( Texture )
    • + *
+ * + * The purpose is to bring the generic model into an OpenGL context + * [Scene] -> [GLScene] + */ +class GLScene(private val scene: Scene) { + + /** + * The wrapped GLModels that are wrapped by this GL scene. + */ + private val models: MutableMap = HashMap() + + /** + * The texture cache that is used by this GL scene. + * Textures are cached to avoid loading the same texture multiple times. + * Has no corresponding generic class. + */ + private val textureCache: GLTextureCache = GLTextureCache() + + init { + updateGlSceneFromScene() + } + + /** + * Adds a model to the scene. + * + * @param model The model that is added to the scene. + */ + fun addModel(model: IModel) { + scene.addModel(model) + updateGlSceneFromScene() + } + + /** + * Updates the GL scene from the scene. + * It updates the GL scene content to match the scene content. + */ + private fun updateGlSceneFromScene() { + scene.getModels().forEach { (k, v) -> + models.putIfAbsent(k, GLModel(v)) + } + models.forEach { (_, v) -> + v.getMaterials()?.forEach { mat -> + if (mat != null) { + textureCache.addTextureIfAbsent(mat.texture) + } + } + } + } + + /** + * Adds an entity to the corresponding model. + * @param entity The entity that is added to the model. + */ + fun addEntity(entity: Entity) { + val modelId = entity.modelId + val model = models[modelId] ?: throw RuntimeException("Model not found: $modelId") + model.addEntity(entity) + } + + /** + * Returns the GL models of the GL scene. + * @return The GL models of the GL scene. + */ + fun getModels(): Map { + return models + } + + /** + * Returns the texture cache of the GL scene. + * @return The texture cache of the GL scene. + */ + fun getTextureCache(): GLTextureCache { + return textureCache + } + + /** + * Returns the projection of the wrapped generic scene. + * @return The projection of the wrapped generic scene. + */ + fun getProjection(): Projection { + return scene.getProjection() + } + + /** + * Returns the camera of the wrapped generic scene. + * @return The camera of the wrapped generic scene. + */ + fun getCamera(): Camera { + return scene.getCamera() + } + + /** + * Clears the models of the GL scene but not containing resources. + * Removes the references to the wrapped generic models and textures. + * Hence, the models could be used by another extraction task this method does not close the models or textures. + * Can be used to only remove models temporarily from GL scene. + */ + fun clearModels() { + cleanup() + models.clear() + } + + /** + * Cleans up the GL scene and calls all underlying cleanup methods. + * Removes only the references to wrapped generic models and textures. + * Hence, the model could be used by another extraction task this method does not close the generic models or textures. + */ + fun cleanup() { + models.values.forEach { it.cleanup() } + models.clear() + textureCache.cleanup() + } + + /** + * Resizes the projection of the wrapped generic scene. + * @param width The new width of the projection. + * @param height The new height of the projection. + */ + fun resize(width: Int, height: Int) { + scene.getProjection().updateProjMatrix(width, height) + } +} diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/glmodel/GLTexture.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/glmodel/GLTexture.kt new file mode 100644 index 000000000..65f7323d6 --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/glmodel/GLTexture.kt @@ -0,0 +1,134 @@ +package org.vitrivr.engine.model3d.lwjglrender.glmodel + +import java.awt.image.BufferedImage +import java.io.File +import java.io.IOException +import java.nio.ByteBuffer +import javax.imageio.ImageIO +import org.lwjgl.BufferUtils +import org.lwjgl.opengl.GL30 +import org.lwjgl.stb.STBImage +import org.lwjgl.system.MemoryStack +import org.vitrivr.engine.core.model.mesh.texturemodel.Texture + +/** + * The GLTexture class is a wrapper for the [Texture] class. + * * Texture -> GLTexture( Texture ) + * + * The purpose is to bring the generic Mesh in an OpenGl context [Texture] -> [GLTexture] + */ +class GLTexture( + /** The wrapped generic texture in gl context */ + private val texture: Texture +) { + /** The id of the texture used to bind the texture to the Gl context */ + private var textureId = 0 + + /** + * Creates a new GLTexture from a texture. + * 1. Load the texture from the file + * 1. Allocate the texture buffer + * 1. Load the texture into the buffer + * + * @param texture The texture that is wrapped by this gl texture. + */ + init { + MemoryStack.stackPush().use { memoryStack -> + val w = memoryStack.mallocInt(1) + val h = memoryStack.mallocInt(1) + val channels = memoryStack.mallocInt(1) + val imageBuffer: ByteBuffer? + if (texture.texturePath != null) { + imageBuffer = STBImage.stbi_load(texture.texturePath, w, h, channels, 4) + } else if (texture.textureImage != null) { + val imagePath = "tmp.png" + val outputFile = File(imagePath) + try { + // Write image to file + ImageIO.write(texture.textureImage, "png", outputFile) + } catch (e: IOException) { + System.err.println("Error saving tmp texture image: " + e.message) + } + imageBuffer = STBImage.stbi_load(imagePath, w, h, channels, 4) + if (outputFile.exists()) { + // Attempt to delete the file + outputFile.delete() + } + } else { + throw RuntimeException("Could not load texture file: " + texture.texturePath) + } + this.generateTexture(w.get(), h.get(), imageBuffer) + STBImage.stbi_image_free(imageBuffer) + } + } + + // TODO Convert the image data to a byte buffer to avoid tmp texture file --> requires further + // debugging as the rendering afterward is not correct + fun convertImageData(image: BufferedImage): ByteBuffer { + val pixels = IntArray(image.width * image.height) + image.getRGB(0, 0, image.width, image.height, pixels, 0, image.width) + + val buffer = BufferUtils.createByteBuffer(image.width * image.height * 3) + + buffer.clear() + for (y in 0 until image.height) { + for (x in 0 until image.width) { + val pixel = pixels[y * image.width + x] + buffer.put(((pixel shr 16) and 0xFF).toByte()) // Red component + buffer.put(((pixel shr 8) and 0xFF).toByte()) // Green component + buffer.put((pixel and 0xFF).toByte()) // Blue component + } + } + buffer.clear() + return buffer + } + + /** Binds the GLTexture to the Gl context */ + fun bind() { + GL30.glBindTexture(GL30.GL_TEXTURE_2D, this.textureId) + } + + /** + * Cleans the GLTexture Does not affect the underlying texture Removes the texture from the GPU + */ + fun cleanup() { + GL30.glDeleteTextures(this.textureId) + } + + /** + * Generates the texture in the Gl context + * + * @param width The width of the texture + * @param height The height of the texture + * @param texture The texture buffer + */ + private fun generateTexture(width: Int, height: Int, texture: ByteBuffer?) { + this.textureId = GL30.glGenTextures() + GL30.glBindTexture(GL30.GL_TEXTURE_2D, this.textureId) + GL30.glPixelStorei(GL30.GL_UNPACK_ALIGNMENT, 1) + GL30.glTexParameteri(GL30.GL_TEXTURE_2D, GL30.GL_TEXTURE_MIN_FILTER, GL30.GL_NEAREST) + GL30.glTexParameteri(GL30.GL_TEXTURE_2D, GL30.GL_TEXTURE_MAG_FILTER, GL30.GL_NEAREST) + GL30.glTexImage2D( + GL30.GL_TEXTURE_2D, + 0, + GL30.GL_RGBA, + width, + height, + 0, + GL30.GL_RGBA, + GL30.GL_UNSIGNED_BYTE, + texture) + GL30.glGenerateMipmap(GL30.GL_TEXTURE_2D) + } + + val texturePath: String? + /** + * Returns the texture path of the underlying wrapped texture + * + * @return The texture path of the underlying wrapped texture + */ + get() = texture.texturePath + + val textureImage: BufferedImage? + get() = texture.textureImage +} diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/glmodel/GLTextureCache.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/glmodel/GLTextureCache.kt new file mode 100644 index 000000000..d6e888bbb --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/glmodel/GLTextureCache.kt @@ -0,0 +1,55 @@ +package org.vitrivr.engine.model3d.lwjglrender.glmodel + +import org.vitrivr.engine.core.model.mesh.texturemodel.Texture +import java.util.function.Consumer + +/** + * A cache for textures + * Prevents the same texture from being loaded multiple times + */ +class GLTextureCache { + /** + * The cache of textures + */ + private val textures: MutableMap = + HashMap() + + /** + * Creates a new texture cache + * Adds a default texture to the cache + */ + init { + val texture = Texture() + textures[texture.texturePath] = GLTexture(texture) + textures["default"] = GLTexture(texture) + } + + /** + * Cleans up the texture cache + * Cleans the registered textures and clears the cache + */ + fun cleanup() { + textures.values.forEach(Consumer { obj: GLTexture -> obj.cleanup() }) + textures.clear() + } + + /** + * Adds a texture to the cache if it is not already present + * + * @param texture Texture to add + */ + fun addTextureIfAbsent(texture: GLTexture) { + textures.putIfAbsent(texture.texturePath, texture) + } + + /** + * Returns the gl texture with the given texture path + * + * @param texturePath Path of the texture + * @return The texture with the given texture path + */ + fun getTexture(texturePath: String?): GLTexture? { + return textures[texturePath] + } +} + diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/glmodel/IGLModel.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/glmodel/IGLModel.kt new file mode 100644 index 000000000..1b2553faf --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/glmodel/IGLModel.kt @@ -0,0 +1,45 @@ +package org.vitrivr.engine.model3d.lwjglrender.glmodel + +import org.vitrivr.engine.core.model.mesh.texturemodel.Entity + +/** + * The Interface IGLModel provides functionality for an arbitrary model used in the OpenGL context. + * It is the context-related counterpart to the [IModel] interface. + */ +interface IGLModel { + + /** + * Returns the entities of the wrapped generic model. + * + * @return The entities of the wrapped generic model. + */ + fun getEntities(): List + + /** + * Adds an entity to the wrapped generic model. + * + * @param entity The entity to be added. + */ + fun addEntity(entity: Entity) + + /** + * Cleans up the GL model and calls all underlying cleanup methods. + * Removes only the references to wrapped generic materials + * Hence, the model could be used by another extraction task this method does not close the generic model. + */ + fun cleanup() + + /** + * Returns the ID of the wrapped generic model. + * + * @return The ID of the wrapped generic model. + */ + fun getId(): String + + /** + * Returns the GL materials of the GL model. + * + * @return The GL materials of the GL model. + */ + fun getMaterials(): List +} diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/render/Render.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/render/Render.kt new file mode 100644 index 000000000..6329cddd0 --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/render/Render.kt @@ -0,0 +1,67 @@ +package org.vitrivr.engine.model3d.lwjglrender.render + +import org.lwjgl.opengl.GL +import org.lwjgl.opengl.GL30 +import org.vitrivr.engine.model3d.lwjglrender.window.Window +import org.vitrivr.engine.model3d.lwjglrender.glmodel.GLScene + + +/** + * This class holds the render logic for the LWJGL engine + * Holds the [SceneRender] which loads shaders + */ +class Render { + /** + * Instance of the scene render + * @see SceneRender + */ + private val sceneRender: SceneRender + + /** + * Instance of the render options + * @see RenderOptions + */ + private var options: RenderOptions? = null + + /** + * Create a render instance Set up the Render options for OpenGL + */ + init { + GL.createCapabilities() + GL30.glEnable(GL30.GL_DEPTH_TEST) + GL30.glEnable(GL30.GL_CULL_FACE) + GL30.glCullFace(GL30.GL_BACK) + this.sceneRender = SceneRender() + } + + /** + * Set the render options [RenderOptions] + * + * @param options see [RenderOptions] + */ + fun setOptions(options: RenderOptions?) { + this.options = options + } + + /** + * Releases all resources + */ + fun cleanup() { + sceneRender.cleanup() + this.options = null + } + + /** + * Renders a given Scene in a Given Window + * + * @param window GL (offscreen) window instance [Window] + * @param scene GL Scene (containing all models) [GLScene] + */ + fun render(window: Window, scene: GLScene?) { + GL30.glClear(GL30.GL_COLOR_BUFFER_BIT or GL30.GL_DEPTH_BUFFER_BIT) + GL30.glViewport(0, 0, window.width, window.height) + if (scene != null) { + this.options?.let { sceneRender.render(scene, it) } + } + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/render/RenderOptions.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/render/RenderOptions.kt new file mode 100644 index 000000000..8c4e23324 --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/render/RenderOptions.kt @@ -0,0 +1,33 @@ +package org.vitrivr.engine.model3d.lwjglrender.render + +import org.vitrivr.engine.core.model.mesh.texturemodel.util.types.Vec4f +import java.io.Serializable +import java.util.function.Function + +/** + * RenderOptions + * + * * Used to switch on or off the texture rendering + * * Used to switch on or off the coloring rendering + * * Returns the color for the given value + * * Can be used to colorize the model custom + */ +data class RenderOptions( + /** + * Used to switch on or off the texture rendering + */ + var showTextures: Boolean = true, + + /** + * Used to switch on or off the coloring rendering For future face coloring + */ + var showColor: Boolean = false, + + /** + * Returns the color for the given value Can be used to colorize the model custom + * + * @TODO: This cannot be serialized! + */ + @Transient + var colorfunction: Function = Function { v: Float? -> Vec4f(v!!, v, v, 1f) } +) : Serializable diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/render/SceneRender.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/render/SceneRender.kt new file mode 100644 index 000000000..176814c58 --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/render/SceneRender.kt @@ -0,0 +1,155 @@ +package org.vitrivr.engine.model3d.lwjglrender.render + +import org.joml.Vector4f +import org.joml.Matrix4f +import org.lwjgl.opengl.GL30 +import org.vitrivr.engine.model3d.lwjglrender.glmodel.GLScene +import org.vitrivr.engine.model3d.lwjglrender.glmodel.GLTexture +import org.vitrivr.engine.model3d.lwjglrender.glmodel.IGLModel +import org.vitrivr.engine.model3d.lwjglrender.render.ShaderProgram.ShaderModuleData + +/** + * SceneRender + * + * * Renders the scene + * * Loads the scene shader + * * Creates the uniforms + * * Binds the Model + * * Binds the Texture + * + */ +class SceneRender { + /** + * Instance of the scene shader program + */ + private val shaderProgram: ShaderProgram + + /** + * Uniforms for the scene shader + */ + private var uniformsMap: UniformsMap? = null + + /** + * SceneRender. During construction: Loads the scene shader from the resources + */ + init { + val shaderModuleDataList = ArrayList() + shaderModuleDataList.add(ShaderModuleData(VERTEX_SHADER_PATH, GL30.GL_VERTEX_SHADER)) + shaderModuleDataList.add(ShaderModuleData(FRAGMENT_SHADER_PATH, GL30.GL_FRAGMENT_SHADER)) + this.shaderProgram = ShaderProgram(shaderModuleDataList) + this.createUniforms() + } + + /** + * Creates the uniforms for the scene shader creates the following uniforms: + * + * * projectionMatrix + * * modelMatrix + * * viewMatrix + * * txtSampler + * * material.diffuse + * + */ + private fun createUniforms() { + this.uniformsMap = UniformsMap(shaderProgram.programId) + uniformsMap!!.createUniform("projectionMatrix") + uniformsMap!!.createUniform("modelMatrix") + uniformsMap!!.createUniform("viewMatrix") + uniformsMap!!.createUniform("txtSampler") + uniformsMap!!.createUniform("material.diffuse") + } + + /** + * Releases all resources + * + * * Releases the shader program + * * Releases the uniforms + * + */ + fun cleanup() { + shaderProgram.cleanup() + uniformsMap!!.cleanup() + this.uniformsMap = null + } + + /** + * Renders the Models in the scene + * + * * Binds projection matrix + * * Binds view matrix + * * Binds texture sampler + * + * Further, iterate over all models in the scene + * + * * Iterate over all materials in the model + * * Sets texture or color function + * * Iterate over all meshes in the material + * * Binds the mesh + * * Iterate over all entities to draw the mesh + * * Binds the model matrix + * * Draws the mesh + * * Unbinds + * + * @param scene Scene to render + * @param opt Render options + */ + /** + * Renders the Models in the scene + * Creates standard render options + * @param scene Scene to render + */ + @JvmOverloads + fun render(scene: GLScene, opt: RenderOptions = RenderOptions()) { + shaderProgram.bind() + + uniformsMap!!.setUniform("projectionMatrix", scene.getProjection().projMatrix) + uniformsMap!!.setUniform("viewMatrix", scene.getCamera().viewMatrix) + uniformsMap!!.setUniform("txtSampler", 0) + + val models: Collection = scene.getModels().values + val textures = scene.getTextureCache() + + for (model in models) { + val entities = model.getEntities() + for (material in model.getMaterials()) { + var texture: GLTexture + // Either draw texture or use color function + if (opt.showTextures) { + val v4f = Vector4f(material.diffuseColor.x, material.diffuseColor.y, material.diffuseColor.z, material.diffuseColor.w) + uniformsMap!!.setUniform("material.diffuse", v4f) + texture = textures.getTexture(material.texture.texturePath)!! + } else { + val v4f = Vector4f(opt.colorfunction.apply(1f).x, opt.colorfunction.apply(1f).y, opt.colorfunction.apply(1f).z, opt.colorfunction.apply(1f).w) + uniformsMap!!.setUniform("material.diffuse", v4f) + texture = textures.getTexture("default")!! + } + GL30.glActiveTexture(GL30.GL_TEXTURE0) + texture.bind() + for (mesh in material.meshes) { + GL30.glBindVertexArray(mesh.vaoId) + for (entity in entities) { + val floatBuffer = entity.modelMatrix.getFloatBuffer() + val matrix4f = Matrix4f(floatBuffer) + uniformsMap!!.setUniform("modelMatrix", matrix4f) + GL30.glDrawElements(GL30.GL_TRIANGLES, mesh.numVertices, GL30.GL_UNSIGNED_INT, 0) + } + } + } + } + GL30.glBindVertexArray(0) + + shaderProgram.unbind() + } + + companion object { + /** + * Resource path to the scene shader program + */ + private const val VERTEX_SHADER_PATH = "/renderer/lwjgl/shaders/scene.vert" + + /** + * Resource path to the fragment shader program + */ + private const val FRAGMENT_SHADER_PATH = "/renderer/lwjgl/shaders/scene.frag" + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/render/ShaderProgram.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/render/ShaderProgram.kt new file mode 100644 index 000000000..9a56b6498 --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/render/ShaderProgram.kt @@ -0,0 +1,173 @@ +package org.vitrivr.engine.model3d.lwjglrender.render + +import org.lwjgl.opengl.GL30 +import java.io.IOException +import java.nio.file.Files +import java.nio.file.Paths +import java.util.function.Consumer + +/** + * ShaderProgram + * Loads a shader program from the resources to the GL context + */ +class ShaderProgram(shaderModuleDataList: List) { + /** + * Returns the program id + * @return program id + */ + /** + * Shader Program ID, is used to bind and release the program from the GL context + */ + val programId: Int = GL30.glCreateProgram() + + /** + * Creates a new ShaderProgram + * Takes a list of ShaderModuleData (usually from Scene Renderer which loads the shaders from the resources during construction + * Creates a new ShaderProgram in GL context, links the shaders and validates the program + * For Shader creation, the following steps are performed: + * + * * Reads the shader file + * * Creates a new shader in the GL context [ShaderProgram.createShader] + * * Compiles the shader + * * Attaches the shader to the program + * * Links the program + * * Binds the program to the GL context + * + * @param shaderModuleDataList List of ShaderModuleData + */ + init { + if (this.programId == 0) { + throw RuntimeException("Could not create shader.") + } + val shaderModules = ArrayList() + shaderModuleDataList.forEach(Consumer { s: ShaderModuleData -> + shaderModules.add( + this.createShader( + readShaderFile(s.shaderFile), + s.shaderType + ) + ) + }) + this.link(shaderModules) + } + + /** + * Binds the ShaderProgram to the GL context + */ + fun bind() { + GL30.glUseProgram(this.programId) + } + + /** + * Unbinds the ShaderProgram from the GL context + */ + fun unbind() { + GL30.glUseProgram(0) + } + + /** + * Unbinds the ShaderProgram from the GL context + * Deletes the ShaderProgram from the GL context + */ + fun cleanup() { + this.unbind() + if (this.programId != 0) { + GL30.glDeleteProgram(this.programId) + } + } + + /** + * Links the program + * Deletes the shaders + * @param shaderModules List of shader ids + */ + private fun link(shaderModules: List) { + GL30.glLinkProgram(this.programId) + if (GL30.glGetProgrami(this.programId, GL30.GL_LINK_STATUS) == 0) { + throw RuntimeException("Error linking Shader") + } + shaderModules.forEach(Consumer { s: Int? -> + GL30.glDetachShader( + this.programId, + s!! + ) + }) + shaderModules.forEach(Consumer { shader: Int? -> + GL30.glDeleteShader( + shader!! + ) + }) + //this.validate(); + } + + /** + * Validates the program + * Throws an exception if the program is not valid + */ + fun validate() { + GL30.glValidateProgram(this.programId) + if (GL30.glGetProgrami(this.programId, GL30.GL_VALIDATE_STATUS) == 0) { + throw RuntimeException("Error validate Shader") + } + } + + /** + * Creates a new Shader in the GL context + * Compiles the shader + * Attaches the shader to the program + * @return the shader id + */ + protected fun createShader(shaderCode: String?, shaderType: Int): Int { + val shaderId = GL30.glCreateShader(shaderType) + check(shaderId != 0) { "Error creating shader" } + GL30.glShaderSource(shaderId, shaderCode) + GL30.glCompileShader(shaderId) + + check(GL30.glGetShaderi(shaderId, GL30.GL_COMPILE_STATUS) != 0) { "Error compiling shader" } + GL30.glAttachShader(this.programId, shaderId) + return shaderId + } + + /** + * RECORD for ShaderModuleData + * @param shaderFile + * @param shaderType + */ + @JvmRecord + data class ShaderModuleData(val shaderFile: String, val shaderType: Int) + companion object { + /** + * Reads the shader file. + * + * @param filePath Path to the shader file + * @return String containing the shader code + */ + fun readShaderFile(filePath: String): String { + var shader: String? = null + + /* First try: Load from resources. */ + try { + ShaderProgram::class.java.getResourceAsStream(filePath).use { stream -> + if (stream != null) { + shader = String(stream.readAllBytes()) + } + } + } catch (e: IOException) { + /* No op. */ + } + + /* Second attempt try: Load from resources. */ + if (shader == null) { + try { + shader = String(Files.readAllBytes(Paths.get(filePath))) + } catch (ex: IOException) { + /* No op. */ + } + } + + /* Make sure shader has been loaded. */ + checkNotNull(shader) { "Error reading shader file: $filePath" } + return shader as String + } + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/render/UniformsMap.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/render/UniformsMap.kt new file mode 100644 index 000000000..bbae06ce7 --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/render/UniformsMap.kt @@ -0,0 +1,92 @@ +package org.vitrivr.engine.model3d.lwjglrender.render + +import org.joml.Matrix4f +import org.joml.Vector4f +import org.lwjgl.opengl.GL30 +import org.lwjgl.system.MemoryStack + +/** + * Holds a hashmap for used uniforms are global variables in the shader e.g. projectionMatrix, modelMatrix, viewMatrix, txtSampler, material.diffuse + * + * @see [](https://www.khronos.org/opengl/wiki/Uniform_ +@see "./resources/renderer/lwjgl/shaders/scene.vert" + * + * @see ["./resources/renderer/lwjgl/shaders/scene.frag"](./resources/renderer/lwjgl/shaders/scene.frag) + */ +class UniformsMap( + /** + * Program id of the shader + */ + private val programId: Int +) { + /** + * HashMap for the uniforms Key: Uniform name Value: Uniform location in the shader + */ + private val uniforms = HashMap() + + /** + * Creates a new uniform + * + * @param uniformName Name of the uniform + */ + fun createUniform(uniformName: String) { + val uniformLocation = GL30.glGetUniformLocation(this.programId, uniformName) + if (uniformLocation < 0) { + throw RuntimeException("Could not find uniform:$uniformName") + } + uniforms[uniformName] = uniformLocation + } + + /** + * Sets the value of a uniform to gl context + * + * @param uniformName Name of the uniform + * @param value Value of the uniform + */ + fun setUniform(uniformName: String, value: Int) { + GL30.glUniform1i(this.getUniformLocation(uniformName), value) + } + + /** + * Returns the location of the uniform from the hashmap + * + * @param uniformName name of the uniform + * @return location of the uniform + */ + private fun getUniformLocation(uniformName: String): Int { + val location = uniforms[uniformName] + ?: throw RuntimeException("Could not find uniform:$uniformName") + return location + } + + /** + * Sets the value 4 float vector of a uniform to gl context + * + * @param uniformName Name of the uniform + * @param value Value of the uniform + */ + fun setUniform(uniformName: String, value: Vector4f) { + GL30.glUniform4f(this.getUniformLocation(uniformName), value.x, value.y, value.z, value.w) + } + + /** + * Sets the value 4*4 float matrix of a uniform to gl context + * + * @param uniformName Name of the uniform + * @param value Value of the uniform + */ + fun setUniform(uniformName: String, value: Matrix4f) { + MemoryStack.stackPush().use { memoryStack -> + val location = uniforms[uniformName] + ?: throw RuntimeException("Could not find uniform:$uniformName") + GL30.glUniformMatrix4fv(location, false, value[memoryStack.mallocFloat(16)]) + } + } + + /** + * Cleans up the uniforms + */ + fun cleanup() { + uniforms.clear() + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/renderer/LWJGLOffscreenRenderer.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/renderer/LWJGLOffscreenRenderer.kt new file mode 100644 index 000000000..5d106e7ad --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/renderer/LWJGLOffscreenRenderer.kt @@ -0,0 +1,228 @@ +package org.vitrivr.engine.model3d.lwjglrender.renderer + +import org.joml.Vector3f +import org.vitrivr.engine.core.model.mesh.texturemodel.Entity +import org.vitrivr.engine.core.model.mesh.texturemodel.IModel +import org.vitrivr.engine.core.model.mesh.texturemodel.Model3d +import org.vitrivr.engine.model3d.lwjglrender.engine.Engine +import org.vitrivr.engine.model3d.lwjglrender.engine.EngineLogic +import org.vitrivr.engine.model3d.lwjglrender.glmodel.GLScene +import org.vitrivr.engine.model3d.lwjglrender.glmodel.IGLModel +import org.vitrivr.engine.model3d.lwjglrender.render.Render +import org.vitrivr.engine.model3d.lwjglrender.render.RenderOptions +import org.vitrivr.engine.model3d.lwjglrender.scene.* +import org.vitrivr.engine.model3d.lwjglrender.window.Window +import org.vitrivr.engine.model3d.lwjglrender.window.WindowOptions +import java.awt.image.BufferedImage +import java.util.concurrent.LinkedTransferQueue +import java.util.function.Consumer + +/** + * This is the top most class of the LWJGL for Java 3D renderer. Its main function is to provide an interface between Engine and the outside world. It extends the abstract class [EngineLogic] which allows the instanced engine to call methods depending on the engine state. + * + * @version 1.0.0 + * @author Raphael Waltenspühl + */ +class LWJGLOffscreenRenderer : EngineLogic() { + /** + * The (offscreen) window options for the engine. + */ + private var windowOptions: WindowOptions? = null + + /** + * The engine instance. + */ + private var engine: Engine? = null + + /** + * The model queue. From this queue the renderer takes the next model to render. + */ + private val modelQueue = LinkedTransferQueue() + + /** + * The image queue. In this queue the renderer puts the rendered images. + */ + private val imageQueue = LinkedTransferQueue() + + /** + * Sets the window options for the engine. + * + * @param opts The window options. + */ + fun setWindowOptions(opts: WindowOptions?) { + this.windowOptions = opts + } + + /** + * Sets the render options for the engine. + * + * @param opts The render options. + */ + fun setRenderOptions(opts: RenderOptions?) { + if (opts != null) { + engine!!.setRenderOptions(opts) + } + } + + /** + * Starts the engine with given window options. Registers the LWJGLOffscreenRenderer as the engine logic. + */ + fun startEngine() { + val name = "LWJGLOffscreenRenderer" + this.engine = this.windowOptions?.let { Engine(name, it, this) } + } + + /** + * Starts the rendering process. + */ + fun render() { + engine!!.runOnce() + } + + /** + * Is called once at the initialization of the engine. DO NOT CALL ENGINE METHODS IN THIS METHOD DO NOT CALL THIS METHOD FROM THIS CLASS + */ + override fun init(window: Window, scene: GLScene, render: Render) { + scene.getCamera().setPosition(0f, 0f, 1f) + } + + /** + * Is called from the engine before the render method. DO NOT CALL ENGINE METHODS IN THIS METHOD DO NOT CALL THIS METHOD FROM THIS CLASS + */ + override fun beforeRender(window: Window, scene: GLScene, render: Render) { + this.loadNextModelFromQueueToScene(window, scene) + } + + /** + * Is called from the engine after the render method. DO NOT CALL ENGINE METHODS IN THIS METHOD DO NOT CALL THIS METHOD FROM THIS CLASS + */ + override fun afterRender(window: Window, scene: GLScene, render: Render) { + val lfc = LightfieldCamera(this.windowOptions!!) + imageQueue.add(lfc.takeLightfieldImage()) + } + + /** + * Is called from the engine as first step during refresh and cleanup DO NOT CALL ENGINE METHODS IN THIS METHOD DO NOT CALL THIS METHOD FROM THIS CLASS + */ + override fun cleanup() { + + } + + + /** + * This method is called every frame. This is only used in continuous rendering. The purpose is to do some input handling. Could be use for optimize view angles on a fast manner. DO NOT CALL ENGINE METHODS IN THIS METHOD DO NOT CALL THIS METHOD FROM THIS CLASS + */ + override fun input(window: Window, scene: GLScene, diffTimeMillis: Long) { + scene.getModels().forEach { (k: String?, v: IGLModel) -> + v.getEntities().forEach( + Consumer { obj: Entity -> obj.updateModelMatrix() }) + } + } + + /** + * After Engine run This method is called every frame. This is only used in continuous rendering. The purpose is to process some life output. Could be use for optimize view angles on a fast manner. DO NOT CALL ENGINE METHODS IN THIS METHOD DO NOT CALL THIS METHOD FROM THIS CLASS + */ + override fun update(window: Window, scene: GLScene, diffTimeMillis: Long) { + } + + /** + * This method is called to load the next model into the provided scene. + * + * @param scene The scene to put the model in. + */ + @Suppress("unused") + private fun loadNextModelFromQueueToScene(window: Window, scene: GLScene) { + if (!modelQueue.isEmpty()) { + val model = modelQueue.poll() as Model3d + if (model.getEntities().isEmpty()) { + val entity = Entity("cube", model.getId()) + model.addEntityNorm(entity) + } + //cleans all current models from the scene + scene.cleanup() + //adds the new model to the scene + scene.addModel(model) + } + scene.getModels().forEach { (k: String?, v: IGLModel) -> + v.getEntities().forEach( + Consumer { obj: Entity -> obj.updateModelMatrix() }) + } + } + + /** + * Moves the camera in the scene with given deltas in cartesian coordinates. Look at the origin. + */ + fun moveCameraOrbit(dx: Float, dy: Float, dz: Float) { + engine!!.getCamera().moveOrbit(dx, dy, dz) + } + + /** + * Sets the camera in the scene to cartesian coordinates. Look at the origin. + */ + fun setCameraOrbit(x: Float, y: Float, z: Float) { + engine!!.getCamera().setOrbit(x, y, z) + } + + /** + * Moves the camera in the scene with given deltas in cartesian coordinates. Keep the orientation. + */ + @Suppress("unused") + fun setCameraPosition(x: Float, y: Float, z: Float) { + engine!!.getCamera().setPosition(x, y, z) + } + + /** + * Set position of the camera and look at the origin. Camera will stay aligned to the y plane. + */ + fun lookFromAtO(x: Float, y: Float, z: Float) { + val lookFrom = Vector3f(x, y, z) + val lookAt = Vector3f(0f, 0f, 0f) + + engine!!.getCamera().setPositionAndOrientation(lookFrom, lookAt) + } + + @get:Suppress("unused") + val aspect: Float + /** + * Returns the aspect ratio of the window. + */ + get() = windowOptions!!.width.toFloat() / windowOptions!!.height.toFloat() + + /** + * Interface to outside to add a model to the scene. + */ + fun assemble(model: IModel) { + modelQueue.add(model) + } + + /** + * Interface to outside to get a rendered image. + */ + fun obtain(): BufferedImage { + return imageQueue.poll() + } + + /** + * This method disposes the engine. Window is destroyed and all resources are freed. + */ + fun clear() { + engine!!.clear() + this.engine = null + } + + val width: Int + /** + * Returns the width of the window. + * + * @return The width of the window. (in pixels) + */ + get() = windowOptions!!.width + + val height: Int + /** + * Returns the height of the window. + * + * @return The height of the window. (in pixels) + */ + get() = windowOptions!!.height +} \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/renderer/RenderActions.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/renderer/RenderActions.kt new file mode 100644 index 000000000..07bfa63d2 --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/renderer/RenderActions.kt @@ -0,0 +1,13 @@ +package org.vitrivr.engine.model3d.lwjglrender.renderer + +/** + * Actions used in the Render Workflow. + * [RenderWorker] + */ +enum class RenderActions(val action: String) { + SETUP("SETUP"), + RENDER("RENDER"), + ROTATE("ROTATE"), + LOOKAT("LOOKAT"), + LOOKAT_FROM("LOOKAT_FROM"); +} diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/renderer/RenderData.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/renderer/RenderData.kt new file mode 100644 index 000000000..a43e20419 --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/renderer/RenderData.kt @@ -0,0 +1,14 @@ +package org.vitrivr.engine.model3d.lwjglrender.renderer + +/** + * Data used of the Render Workflow + * [RenderWorker] + */ +object RenderData { + const val MODEL: String = "MODEL" + const val VECTOR: String = "VECTOR" + const val VECTORS: String = "VECTORS" + const val IMAGE: String = "IMAGE" + const val WINDOWS_OPTIONS: String = "WINDOWS_OPTIONS" + const val RENDER_OPTIONS: String = "RENDER_OPTIONS" +} diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/renderer/RenderJob.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/renderer/RenderJob.kt new file mode 100644 index 000000000..fa905f020 --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/renderer/RenderJob.kt @@ -0,0 +1,162 @@ +package org.vitrivr.engine.model3d.lwjglrender.renderer + +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import org.joml.Vector3f +import org.vitrivr.engine.core.model.mesh.texturemodel.IModel +import org.vitrivr.engine.model3d.lwjglrender.util.datatype.Variant +import org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker.Job +import org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker.JobControlCommand +import org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker.JobType +import org.vitrivr.engine.model3d.lwjglrender.util.fsm.model.Action +import org.vitrivr.engine.model3d.lwjglrender.window.WindowOptions +import org.vitrivr.engine.model3d.lwjglrender.render.RenderOptions +import java.awt.image.BufferedImage +import java.util.* +import java.util.concurrent.BlockingDeque +import java.util.concurrent.LinkedBlockingDeque + +/** + * The RenderJob is a job which is responsible for rendering a model. + * + * + * This job extends the abstract class Job. + * + * + * It provides constructors for the different types of jobs. + * ORDER Job to render a model. + * COMMAND Job signals caller that the job is done or an error occurred. + * RESULT Job contains the result of the rendering. + */ +class RenderJob : Job { + /** + * Creates a new ORDER RenderJob with the given action sequence and data (containing the model to render). + */ + constructor(actions: BlockingDeque?, data: Variant?) : super(actions, data) + + /** + * Creates a new RESPONSE RenderJob with the rendered image. + */ + constructor(data: Variant?) : super(data) + + /** + * Creates a new CONTROL RenderJob with the given command. + */ + constructor(command: JobControlCommand?) : super(command) + + + companion object { + private val LOGGER: Logger = LogManager.getLogger() + + /** + * Static method to create a standard render job. + * + * + */ + fun performStandardRenderJob( + renderJobQueue: BlockingDeque, + model: IModel?, + cameraPositions: Array, + windowOptions: WindowOptions?, + renderOptions: RenderOptions? + ): List { + val cameraPositionVectors = LinkedList() + for (cameraPosition in cameraPositions) { + assert(cameraPosition.size == 3) + cameraPositionVectors.add( + Vector3f( + cameraPosition[0].toFloat(), + cameraPosition[1].toFloat(), + cameraPosition[2].toFloat() + ) + ) + } + return performStandardRenderJob(renderJobQueue, model!!, cameraPositionVectors, windowOptions!!, renderOptions!!) + } + + /** + * Static method to create a standard render job. + * + * + * + * * Creates a job for given model and each camera position. + * * Adds the data to the variant (data bag) + * * Generates the needed actions for the job. + * * Creates the job and adds it to the provided queue. + * * Waits for the job to finish. (or fail) + * * Returns the rendered images. + * * Cleans up the resources. + * + * + * + * @param renderJobQueue The queue to add the job to. + * @param model The model to render. + * @param cameraPositions The camera positions to render the model from. + * @param windowOptions The window options to use for the rendering. + * @param renderOptions The render options to use for the rendering. + * @return The rendered images. + */ + fun performStandardRenderJob( + renderJobQueue: BlockingDeque, + model: IModel, + cameraPositions: LinkedList, + windowOptions: WindowOptions, + renderOptions: RenderOptions + ): List { + // Create data bag for the job. + val jobData = Variant() + jobData.set(RenderData.WINDOWS_OPTIONS, windowOptions) + .set(RenderData.RENDER_OPTIONS, renderOptions) + .set(RenderData.MODEL, model) + + // Setup the action sequence to perform the jop + // In standard jop, this is an image for each camera position + val actions = LinkedBlockingDeque() + + actions.add(Action(RenderActions.SETUP.name)) + actions.add(Action(RenderActions.SETUP.name)) + actions.add(Action(RenderActions.SETUP.name)) + + val vectors = LinkedList() + for (position in cameraPositions) { + // Create a copy of the vector to avoid concurrent modification exceptions + vectors.add(Vector3f(position)) + actions.add(Action(RenderActions.LOOKAT_FROM.name)) + actions.add(Action(RenderActions.RENDER.name)) + } + actions.add(Action(RenderActions.SETUP.name)) + jobData.set(RenderData.VECTORS, vectors) + + // Add the job to the queue + val job = RenderJob(actions, jobData) + renderJobQueue.add(job) + + // Wait for the job to finish + var finishedJob = false + val image = ArrayList() + + // Add images to result or finish the job + try { + while (!finishedJob) { + val result = job.results + if (result.type == JobType.RESPONSE) { + image.add(result.data!!.get(BufferedImage::class.java, RenderData.IMAGE)) + } else if (result.type == JobType.CONTROL) { + if (result.command == JobControlCommand.JOB_DONE) { + finishedJob = true + } + if (result.command == JobControlCommand.JOB_FAILURE) { + LOGGER.error("Job failed") + finishedJob = true + } + } + } + } catch (ex: InterruptedException) { + LOGGER.error("Could not render model", ex) + } finally { + job.clean() + } + return image + } + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/renderer/RenderStates.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/renderer/RenderStates.kt new file mode 100644 index 000000000..09b8e0e55 --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/renderer/RenderStates.kt @@ -0,0 +1,17 @@ +package org.vitrivr.engine.model3d.lwjglrender.renderer + +/** + * States of the Render Workflow + * [RenderWorker] + */ +object RenderStates { + const val IDLE: String = "IDLE" + const val INIT_WINDOW: String = "INIT_WINDOW" + const val INIT_RENDERER: String = "INIT_RENDERER" + const val LOAD_MODEL: String = "INIT_MODEL" + const val RENDER: String = "RENDER" + const val ROTATE: String = "ROTATE" + const val LOOKAT: String = "LOOKAT" + const val LOOK_FROM_AT_O: String = "LOOK_FROM_AT_O" + const val UNLOAD_MODEL: String = "UNLOAD_MODEL" +} diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/renderer/RenderWorker.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/renderer/RenderWorker.kt new file mode 100644 index 000000000..72c59ba36 --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/renderer/RenderWorker.kt @@ -0,0 +1,219 @@ +package org.vitrivr.engine.model3d.lwjglrender.renderer + +import java.util.* +import java.util.concurrent.BlockingDeque +import kotlin.math.pow +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import org.joml.Vector3f +import org.lwjgl.system.Configuration +import org.vitrivr.engine.core.model.mesh.texturemodel.IModel +import org.vitrivr.engine.model3d.lwjglrender.util.datatype.Variant +import org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker.JobControlCommand +import org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker.StateEnter +import org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker.StateProvider +import org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker.Worker +import org.vitrivr.engine.model3d.lwjglrender.util.fsm.model.Action +import org.vitrivr.engine.model3d.lwjglrender.util.fsm.model.Graph +import org.vitrivr.engine.model3d.lwjglrender.util.fsm.model.State +import org.vitrivr.engine.model3d.lwjglrender.util.fsm.model.Transition +import org.vitrivr.engine.model3d.lwjglrender.window.WindowOptions +import org.vitrivr.engine.model3d.lwjglrender.render.RenderOptions + +/** + * The RenderWorker is a worker which is responsible for rendering a model. + * + * This worker implements all methods which are needed to do a RenderJob. + * + * It constructs a Graph which describes the states and transitions which a render worker can do. + * + * If a job throws an exception the worker will send a JobControlCommand.ERROR to the caller. + * Furthermore, the worker will unload the model. + * + * Each rendered image will be sent to the caller. + * + * The worker initializes the LWJGL engine. + * + * @see LWJGLOffscreenRenderer + * @see Worker + */ +@StateProvider +class RenderWorker(jobs: BlockingDeque) : Worker(jobs) { + + private var renderer: LWJGLOffscreenRenderer? = null + + init { + Configuration.STACK_SIZE.set(2.0.pow(17).toInt()) + this.renderer = LWJGLOffscreenRenderer() + val defaultOptions = WindowOptions() + renderer!!.setWindowOptions(defaultOptions) + renderer!!.startEngine() + + renderJobQueue = jobs + LOGGER.trace("Initialized RenderWorker") + } + + companion object { + private val LOGGER: Logger = LogManager.getLogger(RenderWorker::class.java) + private lateinit var renderJobQueue: BlockingDeque + + /** + * Static getter for the renderJobQueue. A caller can use get the queue submit new jobs to the + * render worker. + * + * @return the render job queue + */ + fun getRenderJobQueue(): BlockingDeque { + return renderJobQueue + } + } + + /** The render worker main thread. */ + override fun run() { + super.run() + LOGGER.trace("Running RenderWorker") + } + + /** + * Creates the graph for the RenderWorker. + * + * @return the graph + */ + override fun createGraph(): Graph { + return Graph( + // Setup the graph for the RenderWorker + hashMapOf( + Transition(State(RenderStates.IDLE), Action(RenderActions.SETUP.name)) to State(RenderStates.INIT_WINDOW), + Transition(State(RenderStates.INIT_WINDOW), Action(RenderActions.SETUP.name)) to State(RenderStates.LOAD_MODEL), + Transition(State(RenderStates.LOAD_MODEL), Action(RenderActions.SETUP.name)) to State(RenderStates.INIT_RENDERER), + Transition(State(RenderStates.LOAD_MODEL), Action(RenderActions.RENDER.name)) to State(RenderStates.RENDER), + Transition(State(RenderStates.LOAD_MODEL), Action(RenderActions.LOOKAT.name)) to State(RenderStates.LOOKAT), + Transition(State(RenderStates.LOAD_MODEL), Action(RenderActions.LOOKAT_FROM.name)) to State(RenderStates.LOOK_FROM_AT_O), + Transition(State(RenderStates.INIT_RENDERER), Action(RenderActions.RENDER.name)) to State(RenderStates.RENDER), + Transition(State(RenderStates.INIT_RENDERER), Action(RenderActions.LOOKAT.name)) to State(RenderStates.LOOKAT), + Transition(State(RenderStates.INIT_RENDERER), Action(RenderActions.LOOKAT_FROM.name)) to State(RenderStates.LOOK_FROM_AT_O), + Transition(State(RenderStates.RENDER), Action(RenderActions.ROTATE.name)) to State(RenderStates.ROTATE), + Transition(State(RenderStates.RENDER), Action(RenderActions.LOOKAT.name)) to State(RenderStates.LOOKAT), + Transition(State(RenderStates.RENDER), Action(RenderActions.LOOKAT_FROM.name)) to State(RenderStates.LOOK_FROM_AT_O), + Transition(State(RenderStates.RENDER), Action(RenderActions.SETUP.name)) to State(RenderStates.UNLOAD_MODEL), + Transition(State(RenderStates.ROTATE), Action(RenderActions.RENDER.name)) to State(RenderStates.RENDER), + Transition(State(RenderStates.LOOKAT), Action(RenderActions.RENDER.name)) to State(RenderStates.RENDER), + Transition(State(RenderStates.LOOK_FROM_AT_O), Action(RenderActions.RENDER.name)) to State(RenderStates.RENDER) + ), + State(RenderStates.IDLE), + hashSetOf(State(RenderStates.UNLOAD_MODEL)) + ) + } + + + /** + * Handler for render exceptions. Unloads the model and sends a JobControlCommand.ERROR to the + * caller. + * + * @param ex The exception that was thrown. + * @return The handler message. + */ + override fun onJobException(ex: Exception?): String? { + this.unload() + this.currentJob?.putResultQueue(RenderJob(JobControlCommand.JOB_FAILURE)) + return "Job failed" + } + + /** Initializes the renderer. Sets the window options and starts the engine. */ + @StateEnter(state = RenderStates.INIT_WINDOW, data = [RenderData.WINDOWS_OPTIONS]) + fun setWindowOptions(opt: WindowOptions) { + LOGGER.trace("INIT_WINDOW RenderWorker") + this.renderer = LWJGLOffscreenRenderer() + + renderer!!.setWindowOptions(opt) + renderer!!.startEngine() + } + + /** Sets specific render options. */ + @StateEnter(state = RenderStates.INIT_RENDERER, data = [RenderData.RENDER_OPTIONS]) + fun setRendererOptions(opt: RenderOptions) { + LOGGER.trace("INIT_RENDERER RenderWorker") + this.renderer!!.setRenderOptions(opt) + } + + /** State to wait for new jobs. */ + @StateEnter(state = RenderStates.IDLE) + fun idle() { + LOGGER.trace("IDLE RenderWorker") + } + + /** + * Register a model to the renderer. + * + * @param model The model to register and to be rendered. + */ + @StateEnter(state = RenderStates.LOAD_MODEL, data = [RenderData.MODEL]) + fun registerModel(model: IModel) { + LOGGER.trace("LOAD_MODEL RenderWorker") + this.renderer!!.assemble(model) + } + + /** Renders the model. Sends the rendered image to the caller. */ + @StateEnter(state = RenderStates.RENDER) + fun renderModel() { + LOGGER.trace("RENDER RenderWorker") + this.renderer!!.render() + val pic = this.renderer!!.obtain() + val data = Variant().set(RenderData.IMAGE, pic) + val responseJob = RenderJob(data) + this.currentJob?.putResultQueue(responseJob) + } + + /** + * Rotates the camera. + * + * @param rotation The rotation vector (x,y,z) + */ + @StateEnter(state = RenderStates.ROTATE, data = [RenderData.VECTOR]) + fun rotate(rotation: Vector3f) { + LOGGER.trace("ROTATE RenderWorker") + this.renderer!!.moveCameraOrbit(rotation.x, rotation.y, rotation.z) + } + + /** + * Looks at the origin from a specific position. The rotation is not affected. Removes the + * processed position vector from the list. + * + * @param vectors The list of position vectors + */ + @StateEnter(state = RenderStates.LOOKAT, data = [RenderData.VECTORS]) + fun lookAt(vectors: LinkedList) { + LOGGER.trace("LOOKAT RenderWorker") + val vec = vectors.poll() + requireNotNull(vec) + this.renderer!!.setCameraOrbit(vec.x, vec.y, vec.z) + } + + /** + * Looks from a specific position at the origin. Removes the processed position vector from the + * list. + * + * @param vectors The list of position vectors + */ + @StateEnter(state = RenderStates.LOOK_FROM_AT_O, data = [RenderData.VECTORS]) + fun lookFromAtO(vectors: LinkedList) { + LOGGER.trace("LOOK_FROM_AT_O RenderWorker") + val vec = vectors.poll() + requireNotNull(vec) + this.renderer!!.lookFromAtO(vec.x, vec.y, vec.z) + } + + /** Unloads the model and sends a JobControlCommand.JOB_DONE to the caller. */ + @StateEnter(state = RenderStates.UNLOAD_MODEL) + fun unload() { + LOGGER.trace("UNLOAD_MODEL RenderWorker") + this.renderer!!.clear() + this.renderer = null + val responseJob = RenderJob(JobControlCommand.JOB_DONE) + this.currentJob?.putResultQueue(responseJob) + } + + private fun hashtableOf(vararg pairs: Pair): Hashtable { + return Hashtable().apply { pairs.forEach { (k, v) -> put(k, v) } } + } +} diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/scene/Camera.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/scene/Camera.kt new file mode 100644 index 000000000..6e4da917e --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/scene/Camera.kt @@ -0,0 +1,234 @@ +package org.vitrivr.engine.model3d.lwjglrender.scene + +import org.joml.Matrix4f +import org.joml.Quaternionf +import org.joml.Vector2f +import org.joml.Vector3f + +/** + * Camera data class for the LWJGL renderer. + * + * This class is responsible for the camera position and orientation. It is used to calculate the view matrix for the renderer. + */ +data class Camera( + val position: Vector3f = Vector3f(), + val rotation: Vector2f = Vector2f(), + val viewMatrix: Matrix4f = Matrix4f(), + var orbitRotation: Quaternionf = Quaternionf() +) { + /** + * Helper Vector for X-Axis translation. + */ + val translativX = Vector3f() + + /** + * Helper Vector for Y-Axis translation. + */ + val translativY = Vector3f() + + /** + * Helper Vector for Z-Axis translation. + */ + val translativZ = Vector3f() +} + +/** + * Rotates the camera by the given amount. + * + * Camera stays aligned to the y plane. + * + * @param x Amount (rad) of rotation around the X-Axis. + * @param y Amount (rad) of rotation around the Y-Axis. + */ +fun Camera.rotate(x: Float, y: Float) { + this.rotation.add(x, y) + this.recalculate() +} + +/** + * Moves the camera by the given amount. + * + * @param x Amount of movement along the X-Axis. + is right, - is left. + * @param y Amount of movement along the Y-Axis. + is up, - is down. + * @param z Amount of movement along the Z-Axis. + is forward, - is backward. + */ +fun Camera.move(x: Float, y: Float, z: Float) { + this.position.add(x, y, z) + when { + x > 0 -> this.move(x, Direction.RIGHT) + x < 0 -> this.move(x, Direction.LEFT) + } + when { + y > 0 -> this.move(y, Direction.UP) + y < 0 -> this.move(y, Direction.DOWN) + } + when { + z > 0 -> this.move(z, Direction.FORWARD) + z < 0 -> this.move(z, Direction.BACKWARD) + } +} + +/** + * Moves the camera by the given amount in the given direction. + * + * @param inc Amount of movement. + * @param direction Direction of movement. + */ +fun Camera.move(inc: Float, direction: Direction) { + when (direction) { + Direction.FORWARD -> { + this.viewMatrix.positiveZ(this.translativZ).negate().mul(inc) + this.position.add(this.translativZ) + } + Direction.BACKWARD -> { + this.viewMatrix.positiveZ(this.translativZ).negate().mul(inc) + this.position.sub(this.translativZ) + } + Direction.LEFT -> { + this.viewMatrix.positiveX(this.translativX).mul(inc) + this.position.sub(this.translativX) + } + Direction.RIGHT -> { + this.viewMatrix.positiveX(this.translativX).mul(inc) + this.position.add(this.translativX) + } + Direction.UP -> { + this.viewMatrix.positiveY(this.translativY).mul(inc) + this.position.add(this.translativY) + } + Direction.DOWN -> { + this.viewMatrix.positiveY(this.translativY).mul(inc) + this.position.sub(this.translativY) + } + } + this.recalculate() +} + +/** + * Recalculates the view matrix. + */ +fun Camera.recalculate() { + this.viewMatrix.identity() + .rotate(this.orbitRotation) + .translate(-this.position.x, -this.position.y, -this.position.z) +} + +/** + * Sets the absolute position of the camera. + * + * @param x Position along the X-Axis. + * @param y Position along the Y-Axis. + * @param z Position along the Z-Axis. + * @return this + */ +fun Camera.setPosition(x: Float, y: Float, z: Float): Camera { + this.position.set(x, y, z) + this.recalculate() + return this +} + +/** + * Sets the absolute position of the camera. + * + * @param position Position of the camera. (x,y,z) + * @return this + */ +fun Camera.setPosition(position: Vector3f): Camera { + return this.setPosition(position.x, position.y, position.z) +} + +/** + * Sets the absolute rotation of the camera. + * + * @param x (rad) Rotation around the X-Axis. + * @param y (rad) Rotation around the Y-Axis. + * @return this + */ +fun Camera.setRotation(x: Float, y: Float): Camera { + this.rotation.set(x, y) + this.recalculate() + return this +} + +/** + * Moves the orbit of the camera by the given amount. + * + * @param x Amount (rad) of rotation around the X-Axis. + * @param y Amount (rad) of rotation around the Y-Axis. + * @param z Amount (rad) of rotation around the Z-Axis. + */ +fun Camera.moveOrbit(x: Float, y: Float, z: Float): Camera { + val adjustedX = (x.toDouble() * 2.0 * Math.PI).toFloat() + val adjustedY = (y.toDouble() * 2.0 * Math.PI).toFloat() + val adjustedZ = (z.toDouble() * 2.0 * Math.PI).toFloat() + this.orbitRotation.rotateYXZ(adjustedY, adjustedX, adjustedZ) + this.recalculate() + return this +} + +/** + * Sets the absolute orbit of the camera. + * + * @param x (rad) Rotation around the X-Axis. + * @param y (rad) Rotation around the Y-Axis. + * @param z (rad) Rotation around the Z-Axis. + * @return this + */ +fun Camera.setOrbit(x: Float, y: Float, z: Float): Camera { + val adjustedX = (x.toDouble() * 2.0 * Math.PI).toFloat() + val adjustedY = (y.toDouble() * 2.0 * Math.PI).toFloat() + val adjustedZ = (z.toDouble() * 2.0 * Math.PI).toFloat() + this.orbitRotation.rotationXYZ(adjustedX, adjustedY, adjustedZ) + this.recalculate() + return this +} + +/** + * Sets the position and the point the camera is looking at. + * + * @param cameraPosition Position of the camera. (x,y,z) + * @param objectPosition Position of the point the camera is looking at. + * @return this + */ +fun Camera.setPositionAndOrientation(cameraPosition: Vector3f, objectPosition: Vector3f): Camera { + val lookDir = Vector3f(objectPosition).sub(cameraPosition).normalize() + val yNorm = Vector3f(0f, 1f, 0f).normalize() + var right = Vector3f(lookDir).cross(yNorm).normalize() + if (java.lang.Float.isNaN(right.x()) || java.lang.Float.isNaN(right.y()) || java.lang.Float.isNaN(right.z())) { + right = Vector3f(1f, 0f, 0f) + } + val up = Vector3f(right).cross(lookDir).normalize() + this.position.set(cameraPosition) + this.orbitRotation = Quaternionf().lookAlong(lookDir, up) + this.recalculate() + return this +} + +/** + * Sets the position and the point the camera is looking at. Furthermore, it sets the up vector of the camera. + * + * @param cameraPosition Position of the camera. + * @param objectPosition Position of the point the camera is looking at. + * @param up Up vector of the camera. + * @return this + */ +fun Camera.setPositionAndOrientation(cameraPosition: Vector3f, objectPosition: Vector3f, up: Vector3f): Camera { + val lookDir = Vector3f(objectPosition).sub(cameraPosition).normalize() + this.position.set(cameraPosition) + this.orbitRotation.lookAlong(lookDir, up) + this.recalculate() + return this +} + +/** + * Helper method to handle the degrees over 360 and under 0. + */ +fun Camera.degreeHandler(degree: Float): Float { + var adjustedDegree = degree + if (adjustedDegree > 360) { + adjustedDegree -= 360f + } else if (adjustedDegree < 0) { + adjustedDegree += 360f + } + return adjustedDegree +} diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/scene/Direction.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/scene/Direction.kt new file mode 100644 index 000000000..1ef8ba888 --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/scene/Direction.kt @@ -0,0 +1,13 @@ +package org.vitrivr.engine.model3d.lwjglrender.scene + +/** + * Enumerates the possible directions in which a [Camera] can move. + */ +enum class Direction { + FORWARD, + BACKWARD, + LEFT, + RIGHT, + UP, + DOWN +} \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/scene/LightfieldCamera.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/scene/LightfieldCamera.kt new file mode 100644 index 000000000..87d212bf2 --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/scene/LightfieldCamera.kt @@ -0,0 +1,86 @@ +package org.vitrivr.engine.model3d.lwjglrender.scene + +import org.lwjgl.BufferUtils +import org.lwjgl.opengl.GL30 +import org.vitrivr.engine.model3d.lwjglrender.window.WindowOptions +import java.awt.image.BufferedImage +import java.nio.FloatBuffer + +/** + * The LightfieldCamera class is used to take a picture of the current rendered scene. + * The picture is stored as a BufferedImage. + */ +class LightfieldCamera( + /** + * The WindowOptions class is used to set the width and height of the resulting image. + */ + private val opts: WindowOptions +) { + /** + * The BufferedImage that is used to store the image data. + */ + private val lightfieldImage = BufferedImage(opts.width, opts.height, BufferedImage.TYPE_INT_RGB) + + /** + * The FloatBuffer from openGL that holds the image data. + */ + private val imageData: FloatBuffer = BufferUtils.createFloatBuffer(opts.width * opts.height * 3) + + + /** + * Initializes the LightfieldCamera with the given WindowOptions. + * Creates a new BufferedImage with the given width and height. + * Reads the image data from the current openGL context. + * @param opts The WindowOptions that are used to set the width and height of the resulting image. + */ + init { + GL30.glReadPixels(0, 0, opts.width, opts.height, GL30.GL_RGB, GL30.GL_FLOAT, this.imageData) + imageData.rewind() + } + + /** + * Takes a picture of the current rendered scene. + * Updates the image data of the BufferedImage. + * Returns the image data as a BufferedImage. + * @return The RenderedScene as a BufferedImage. + */ + fun takeLightfieldImage(): BufferedImage { + this.takePicture() + return this.lightfieldImage + } + + /** + * This method start calculating the pixels of the BufferedImage. + */ + private fun takePicture() { + lightfieldImage.setRGB(0, 0, opts.width, opts.height, this.rgbData, 0, opts.width) + } + + + private val rgbData: IntArray + /** + * This method converts the pixels of the BufferedImage. + * R, G, B values are merged into one int value. + * E.g. + * 
+         * R = 0xAA -> AA0000, G = 0xBB -> 0x00BB00, B = 0xCC -> 0x0000CC
+         * R+ G + B = 0xAABBCC
+
* + * + * @return The image data as an int array. + */ + get() { + val rgbArray = IntArray(opts.height * opts.width) + + for (y in 0 until opts.height) { + for (x in 0 until opts.width) { + val r = (imageData.get() * 255).toInt() shl 16 + val g = (imageData.get() * 255).toInt() shl 8 + val b = (imageData.get() * 255).toInt() + val i = ((opts.height - 1) - y) * opts.width + x + rgbArray[i] = r + g + b + } + } + return rgbArray + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/scene/Projection.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/scene/Projection.kt new file mode 100644 index 000000000..7687618b7 --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/scene/Projection.kt @@ -0,0 +1,57 @@ +package org.vitrivr.engine.model3d.lwjglrender.scene + +import org.joml.Matrix4f + +/** + * The Projection class is used to create a projection matrix. + * The projection matrix is used to transform the 3D scene into a 2D image. + */ +class Projection(width: Int, height: Int) { + /** + * Returns the projection matrix. + * + * @return The projection matrix. + */ + /** + * The projMatrix for rendering the scene. + */ + val projMatrix: Matrix4f = Matrix4f() + + /** + * Initializes the Projection with the given width and height. + * Creates a new projection matrix. + * + * @param width The width of the window. + * @param height The height of the window. + */ + init { + this.updateProjMatrix(width, height) + } + + /** + * Updates the projection matrix. + * + * @param width The width of the window. + * @param height The height of the window. + */ + fun updateProjMatrix(width: Int, height: Int) { + projMatrix.setPerspective(FOV, width.toFloat() / height.toFloat(), Z_NEAR, Z_FAR) + } + + companion object { + /** + * The FOV is the field of view of the camera. + */ + private val FOV = Math.toRadians(60.0).toFloat() + + /** + * The Z_FAR and Z_NEAR values are used to set the clipping planes. + */ + private const val Z_FAR = 100f + + /** + * The Z_FAR and Z_NEAR values are used to set the clipping planes. + */ + private const val Z_NEAR = 0.01f + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/scene/Scene.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/scene/Scene.kt new file mode 100644 index 000000000..6fc9d830b --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/scene/Scene.kt @@ -0,0 +1,69 @@ +package org.vitrivr.engine.model3d.lwjglrender.scene + +import org.vitrivr.engine.core.model.mesh.texturemodel.Entity +import org.vitrivr.engine.core.model.mesh.texturemodel.IModel +import org.vitrivr.engine.model3d.lwjglrender.scene.Camera +import org.vitrivr.engine.model3d.lwjglrender.scene.Projection + +/** + * The Scene class holds generic 3D scene elements (models, etc.). + * A scene consists of a model, a camera, and a projection. + */ +class Scene(width: Int, height: Int) { + + /** + * Map of generic models in the scene. + */ + private val models: MutableMap = mutableMapOf() + + /** + * Projection of the scene. + */ + private val projection: Projection = Projection(width, height) + + /** + * Camera of the scene. + */ + private val camera: Camera = Camera() + + /** + * Add an entity to the corresponding model. + * Can be used to resize the scene before GL context is created. + */ + fun addEntity(entity: Entity) { + val modelId = entity.modelId + val model = models[modelId] + requireNotNull(model) { "Model not found: $modelId" } + model.addEntity(entity) + } + + /** + * Add a model to the scene. + */ + fun addModel(model: IModel) { + models[model.getId()] = model + } + + /** + * Get a model from the scene. + */ + fun getModels(): Map = models + + /** + * Get the projection of the scene. + */ + fun getProjection(): Projection = projection + + /** + * Get the camera of the scene. + */ + fun getCamera(): Camera = camera + + /** + * Resizes the scene. + * Can be used to resize the scene before GL context is created. + */ + fun resize(width: Int, height: Int) { + projection.updateProjMatrix(width, height) + } +} diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/util/MeshMathUtil.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/MeshMathUtil.kt similarity index 79% rename from vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/util/MeshMathUtil.kt rename to vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/MeshMathUtil.kt index c216fe48d..e8a1514b4 100644 --- a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/util/MeshMathUtil.kt +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/MeshMathUtil.kt @@ -1,8 +1,9 @@ -package org.vitrivr.engine.model3d.util +package org.vitrivr.engine.model3d.lwjglrender.util import org.joml.Vector3f import org.joml.Vector3fc -import org.vitrivr.engine.core.model.mesh.Mesh +import org.vitrivr.engine.core.model.mesh.texturemodel.Mesh + import kotlin.math.sqrt /** @@ -36,9 +37,9 @@ object MeshMathUtil { */ fun farthestVertex(mesh: Mesh, point: Vector3fc): Mesh.Vertex { var max: Mesh.Vertex = mesh.getVertex(0) - var dsqMax = point.distanceSquared(max.position) + var dsqMax = point.distanceSquared(Vector3f(max.position.x, max.position.y, max.position.z)) for (v in mesh.vertices()) { - val dsq = point.distanceSquared(v.position) + val dsq = point.distanceSquared(Vector3f(v.position.x, v.position.y, v.position.z)) if (dsq > dsqMax) { dsqMax = dsq max = v @@ -56,9 +57,9 @@ object MeshMathUtil { */ fun closestVertex(mesh: Mesh, point: Vector3fc): Mesh.Vertex { var min: Mesh.Vertex = mesh.getVertex(0) - var dsqMin = point.distanceSquared(min.position) + var dsqMin = point.distanceSquared(Vector3f(min.position.x, min.position.y, min.position.z)) for (v in mesh.vertices()) { - val dsq = point.distanceSquared(v.position) + val dsq = point.distanceSquared(Vector3f(v.position.x, v.position.y, v.position.z)) if (dsq < dsqMin) { dsqMin = dsq min = v @@ -79,7 +80,8 @@ object MeshMathUtil { for (face in mesh.faces()) { val area: Double = face.area if (area > 0.0) { - barycenter.add(face.centroid.mul(area.toFloat())) + val vec3f = face.centroid.mul(area.toFloat()) + barycenter.add(Vector3f(vec3f.x, vec3f.y, vec3f.z)) total += area } } @@ -96,13 +98,19 @@ object MeshMathUtil { */ fun bounds(mesh: Mesh): FloatArray { /* Extract all vertices that are part of a face. */ - val vertices: MutableList = ArrayList(mesh.numberOfVertices) - for (face in mesh.faces()) { - for (vertex in face.vertices) { - vertices.add(vertex.position) + val vertices: MutableList = ArrayList(mesh.getNumVertices()) + for (face in mesh.getNormals()) { + for (i in face as List) { + vertices.add(Vector3f(i.x, i.y, i.z)) } } - return bounds(mesh.vertexPositions) + val list: MutableList = ArrayList(mesh.getNormals().size) + + for ((idx, i) in mesh.getNormals().withIndex()){ + val vector3f = Vector3f(i.x, i.y, i.z) + list[idx] = vector3f + } + return bounds(list) } /** diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/datatype/Variant.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/datatype/Variant.kt new file mode 100644 index 000000000..134142911 --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/datatype/Variant.kt @@ -0,0 +1,69 @@ +package org.vitrivr.engine.model3d.lwjglrender.util.datatype + +/** + * A variant class that can hold any type of object To store an object in the variant use the set method and the generic identifier of the type e.g. variant.set("name", model); To retrieve an object from the variant use the get method and the generic identifier of the type e.g. var model = variant.get("name"); The variant class can also be merged with other variants. + */ +class Variant { + private val variants: MutableMap = HashMap() + + + /** + * Add a value of type T to the variant If T is a variant, all values of the variant are added to the current variant + * + * @param key Key of the value If the key already exists, an exception is thrown + * @param value Value to add + */ + fun set(key: String, value: T): Variant { + try { + if (value is Variant) { + variants.putAll((value as Variant).variants) + } else { + variants[key] = value + } + } catch (ex: IllegalArgumentException) { + throw VariantException("Key already exists") + } + return this + } + + /** + * Get the value stored under the given key from the variant. + * If the key does not exist, an exception is thrown + * If the type of the value does not match the type of T, an exception is thrown + */ + @Throws(VariantException::class) + fun get(clazz: Class, key: String): T { + val `val` = variants[key] + val result: T + try { + result = clazz.cast(`val`) + } catch (ex: Exception) { + throw VariantException("type mismatch" + `val`!!.javaClass, ex) + } + return result + } + + /** + * Get the value stored under the given key from the variant and remove it from the variant. + * If the key does not exist, an exception is thrown + * If the type of the value does not match the type of T, an exception is thrown + */ + @Throws(VariantException::class) + fun remove(clazz: Class, key: String): T { + val `val` = variants.remove(key) + val result: T + try { + result = clazz.cast(`val`) + } catch (ex: ClassCastException) { + throw VariantException("type mismatch") + } + return result + } + + /** + * Clears the variant + */ + fun clear() { + variants.clear() + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/datatype/VariantException.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/datatype/VariantException.kt new file mode 100644 index 000000000..604e4e9f9 --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/datatype/VariantException.kt @@ -0,0 +1,22 @@ +package org.vitrivr.engine.model3d.lwjglrender.util.datatype + +import java.io.Serial + +/** + * Exception thrown by the [Variant] class. + * Used to indicate that a value could not be converted to the requested type. + * Used to indicate that a key is not valid. + */ +@Suppress("unused") +class VariantException : RuntimeException { + constructor(message: String?) : super(message) + + constructor(message: String?, cause: Throwable?) : super(message, cause) + + constructor(cause: Throwable?) : super(cause) + + companion object { + @Serial + private val serialVersionUID = 3713210701207037554L + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/abstractworker/Job.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/abstractworker/Job.kt new file mode 100644 index 000000000..6739719ef --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/abstractworker/Job.kt @@ -0,0 +1,185 @@ +package org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker + +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import org.vitrivr.engine.model3d.lwjglrender.util.datatype.Variant +import org.vitrivr.engine.model3d.lwjglrender.util.fsm.model.Action +import java.util.concurrent.BlockingDeque +import java.util.concurrent.LinkedBlockingDeque + +/** + * A Job is a container for data and actions. A Job can be of type ORDER, RESPONSE or CONTROL + * + * * ORDER: Needs action, data and provides a result queue + * * RESPONSE: A job which contains data which the worker calculated + * * CONTROL: A job which contains a command e.g. end of job or error etc. + * + * + * + * [JobType] + * + * + * If a job is a CONTROL job it contains a command + * [org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker.JobControlCommand] + */ +abstract class Job { + /** + * The queue contains the action sequence of the job that the worker has to perform. + * The actions can be added before registering the job to the worker or on the fly. + * @return The actions queue of the job. + */ + /** + * The actions to perform These actions are performed by the worker + * + * + * The actions can be added before registering the job to the worker + * + * + * The actions can also be added on the fly + */ + val actions: BlockingDeque? + + /** + * The result queue is used to provide results in a ORDER job + */ + private val resultQueue: BlockingDeque? + /** + * @return The data of the job. Null if the job is a CONTROL job + */ + /** + * Sets the data of the job as a variant. + * The key of the data has to match the data string in the annotation. + * @see StateEnter, + * + * @see StateLeave + * + * @see StateTransition + * On a transitions in the StateMachine, the parser search the data that has to be passed to the invoked method by its key and hand it over as a parameter. + * + * @param data The data to set + */ + /** + * The data to process The resulting data + */ + var data: Variant? = null + /** + * The type of the job is used to control the worker + * Order jobs are something the worker has to do. + * Response jobs is the result the worker has done. + * Control jobs are commands for the worker. + * @return The type of the job. + */ + /** + * The type of the job The type of the job can be ORDER, RESPONSE or CONTROL [JobType] + */ + val type: JobType + /** + * The command is the detailed information of a control job. + * @return The command of the job + */ + /** + * The command of the job if the job is a control job The command of the job can be END, ERROR or NONE [org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker.JobControlCommand] + */ + val command: JobControlCommand? + + /** + * Creates a new ORDER Job to perform a task. + * + * + * ORDER: Needs actions, data and provides a result queue + * + * @param actions The actions to perform + * @param data The data to process + */ + protected constructor(actions: BlockingDeque?, data: Variant?) { + this.command = null + this.actions = actions + this.data = data + this.type = JobType.ORDER + this.resultQueue = LinkedBlockingDeque() + } + + /** + * Creates a new Response Job. + * + * + * RESPONSE: contains data which the worker calculated + * + * @param data The resulting data + */ + protected constructor(data: Variant?) { + this.actions = null + this.command = null + this.data = data + this.type = JobType.RESPONSE + this.resultQueue = null + } + + /** + * Creates a new Control Job + * + * + * CONTROL: contains a command e.g. end of job or error etc. + * @param command The command of the job + */ + protected constructor(command: JobControlCommand?) { + this.actions = null + this.command = command + this.type = JobType.CONTROL + this.resultQueue = null + } + + @get:Throws(InterruptedException::class) + val results: Job + /** + * Returns the result of the job + * + * + * This method blocks until a result is available + * + * @return The result of the job + * @throws InterruptedException If the thread is interrupted while waiting for a result + */ + get() { + checkNotNull(this.resultQueue) + return resultQueue.take() + } + + /** + * Puts a result into the result queue + * + * @param job The result to put into the result queue + */ + fun putResultQueue(job: Job) { + try { + checkNotNull(this.resultQueue) + resultQueue.put(job) + } catch (ex: InterruptedException) { + LOGGER.error("Error while putting result into result queue", ex) + } + } + + /** + * Cleans the job + * + * + * This method should be called after the job is processed + * + * + * It does not affect data in the variant + */ + fun clean() { + data!!.clear() + this.data = null + if (this.actions != null) { + actions.clear() + } + if (this.resultQueue != null) { + resultQueue.clear() + } + } + + companion object { + private val LOGGER: Logger = LogManager.getLogger() + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/abstractworker/JobControlCommand.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/abstractworker/JobControlCommand.kt new file mode 100644 index 000000000..2e5c3b17e --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/abstractworker/JobControlCommand.kt @@ -0,0 +1,23 @@ +package org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker + + +/** + * JobControlCommand are used to control the worker thread. + */ +enum class JobControlCommand { + /** + * Is used to return the end of the job + */ + JOB_DONE, + + /** + * Is used to return an error on the job + */ + JOB_FAILURE, + + + /** + * Is used to shut down the worker + */ + SHUTDOWN_WORKER, +} \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/abstractworker/JobType.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/abstractworker/JobType.kt new file mode 100644 index 000000000..008ed4ccf --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/abstractworker/JobType.kt @@ -0,0 +1,28 @@ +package org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker + +/** + * JobType is used to describe the content of a job + * + * * ORDER: for a job that needs to be performed by the worker + * * RESPONSE: to send the result of a job back to the caller + * * CONTROL: to control the states of worker and caller + * + */ +enum class JobType { + /** + * An Order job contains a request to perform a specific sequence of actions by the worker. + */ + ORDER, + + /** + * Result jobs containing result data e.g. a rendered image + */ + RESPONSE, + + /** + * Control jobs containing a control command, e.g. end of job + * If the job is a control job, the job must contain a JobControlCommand + * @see JobControlCommand + */ + CONTROL, +} \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/abstractworker/StateEnter.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/abstractworker/StateEnter.kt new file mode 100644 index 000000000..b0fdacdc3 --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/abstractworker/StateEnter.kt @@ -0,0 +1,12 @@ +package org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker + +/** + * With this annotation a method can be marked as a state enter method. + * The method will be called when the state is entered. + * The method must have the same number of parameters as the data array. + * The parameters must be in the data container with the same key. + * `@StateEnter(state = "STATENAME", data = {"dataKey1", "dataKey2"} ` + */ +@Retention(AnnotationRetention.RUNTIME) +@Target(AnnotationTarget.FUNCTION, AnnotationTarget.PROPERTY_GETTER, AnnotationTarget.PROPERTY_SETTER) +annotation class StateEnter(val state: String, val data: Array = []) \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/abstractworker/StateLeave.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/abstractworker/StateLeave.kt new file mode 100644 index 000000000..1a3c2ac90 --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/abstractworker/StateLeave.kt @@ -0,0 +1,12 @@ +package org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker + +/** + * With this annotation a method can be marked as a state leave method. + * The method will be called when the state is left. + * The method must have the same number of parameters as the data array. + * The parameters must be in the data container with the same key. + * `@StateLeave(state = "STATENAME", data = {"dataKey1", "dataKey2"} ` + */ +@Retention(AnnotationRetention.RUNTIME) +@Target(AnnotationTarget.FUNCTION, AnnotationTarget.PROPERTY_GETTER, AnnotationTarget.PROPERTY_SETTER) +annotation class StateLeave(val state: String, val data: Array = []) \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/abstractworker/StateProvider.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/abstractworker/StateProvider.kt new file mode 100644 index 000000000..6e4f26d57 --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/abstractworker/StateProvider.kt @@ -0,0 +1,9 @@ +package org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker + + +/** + * With this annotation a class can be marked as a state provider. + */ +@Retention(AnnotationRetention.RUNTIME) +@Target(AnnotationTarget.CLASS) +annotation class StateProvider \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/abstractworker/StateProviderAnnotationParser.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/abstractworker/StateProviderAnnotationParser.kt new file mode 100644 index 000000000..b5844ab6f --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/abstractworker/StateProviderAnnotationParser.kt @@ -0,0 +1,199 @@ +package org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker + +import org.vitrivr.engine.model3d.lwjglrender.util.datatype.Variant +import org.vitrivr.engine.model3d.lwjglrender.util.fsm.model.Action +import org.vitrivr.engine.model3d.lwjglrender.util.fsm.model.State +import org.vitrivr.engine.model3d.lwjglrender.util.fsm.model.Transition +import java.lang.reflect.InvocationTargetException +import java.lang.reflect.Method +import java.util.* + +/** + * This class is used to parse the annotations of a state provider + * @see StateProviderAnnotationParser.runTransitionMethods + */ +class StateProviderAnnotationParser() { + /** + * This method is used to invoke annotated methods of a state provider + * It invokes all of provided Object methods which are annotated with [StateTransition], [org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker.StateEnter] or [org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker.StateLeave] + * For this purpose the caller has to provide the current state, the state which is leaved and the current transition + * The sequence of the method invocation is the following: + * + * 1. Check if the object is a state provider (has the [org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker.StateProvider] annotation) + * 1. Get all methods which are annotated with [StateTransition], [org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker.StateEnter] or [org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker.StateLeave] + * 1. Invoke the methods with the provided data + * + * + * @see org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker.StateProvider + * + * @see StateTransition + * + * @see org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker.StateEnter + * + * @see org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker.StateLeave + * + * + * @param object the instance of the state provider + * @param leavedState the state which is leaved + * @param enteredState the state which is entered + * @param currentTransition the current transition (a state action pair) + * @param data the data which is passed to the state provider methods + * @throws InvocationTargetException if the method cannot be invoked + * @throws IllegalAccessException if the method cannot be accessed + */ + @Throws(InvocationTargetException::class, IllegalAccessException::class) + fun runTransitionMethods( + `object`: Any, + leavedState: State, + enteredState: State, + currentTransition: Transition, + data: Variant + ) { + this.checkIfStateProvider(`object`) + val methods = this.getTransitionRelatedMethods(`object`, leavedState, enteredState, currentTransition) + for (method: Method in methods) { + val params = ArrayList() + val paramNames = getMethodRelatedParams(method) + for (name: String? in paramNames) { + params.add(data.get(Any::class.java, name!!)) + } + method.isAccessible = true + try { + method.invoke(`object`, *params.toTypedArray()) + } catch (ex: IllegalArgumentException) { + throw StateProviderException("The method " + method.name + " has the wrong parameters" + params) + } + } + } + + /** + * Checks if the provided object is a state provider + * @param object the object as instance of worker implementation + * @throws StateProviderException if the object is not a state provider + */ + @Throws(StateProviderException::class) + private fun checkIfStateProvider(`object`: Any) { + if (Objects.isNull(`object`)) { + throw StateProviderException("StateProvider is null") + } + val clazz: Class<*> = `object`.javaClass + if (!clazz.isAnnotationPresent(StateProvider::class.java)) { + throw StateProviderException( + "The class " + + clazz.simpleName + + " is not a state provider" + ) + } + } + + /** + * Returns all methods which are annotated with [StateTransition], [org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker.StateEnter] or [org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker.StateLeave] + * such that the method is related to the provided state and transition. + * Which means that the method has to be invoked on this state transition + * @param object the instance of the state provider + * @param leavedState the state which is leaved + * @param enteredState the state which is entered + * @param currentTransition the current transition (a state action pair) + * @return All methods which have to be invoked on this state transition + */ + private fun getTransitionRelatedMethods( + `object`: Any, leavedState: State, enteredState: State, + currentTransition: Transition + ): List { + val methods = LinkedList() + val clazz: Class<*> = `object`.javaClass + for (method: Method in clazz.declaredMethods) { + if (this.shouldInvokeMethod(method, leavedState, enteredState, currentTransition)) { + methods.add(method) + } + } + return methods + } + + /** + * Helper method which returns true if a given method is related to the provided state and transition + * Which means that the method has to be invoked on this state transition + * @param method the method which is checked + * @param leavedState the state which is leaved + * @param enteredState the state which is entered + * @param currentTransition the current transition (a state action pair) + * @return true if the method has to be invoked on this state transition + */ + private fun shouldInvokeMethod( + method: Method, leavedState: State, enteredState: State, + currentTransition: Transition + ): Boolean { + if (method.isAnnotationPresent(StateTransition::class.java)) { + val at = method.getAnnotation( + StateTransition::class.java + ) + return this.shouldInvokeMethod(at, currentTransition) + } else if (method.isAnnotationPresent(StateEnter::class.java)) { + val at = method.getAnnotation( + StateEnter::class.java + ) + return this.shouldInvokeMethod(at, enteredState) + } else if (method.isAnnotationPresent(StateLeave::class.java)) { + val at = method.getAnnotation( + StateLeave::class.java + ) + return this.shouldInvokeMethod(at, leavedState) + } + return false + } + + /** + * Helper method for StateTransition annotation + * @param at the annotation of the method + * @param currentTransition the current transition (a state action pair) + * @return true if the method has to be invoked on this transition + */ + private fun shouldInvokeMethod(at: StateTransition, currentTransition: Transition): Boolean { + return Transition(State(at.state), Action(at.action)).equals(currentTransition) + } + + /** + * Helper method for StateEnter annotation + * @param at the annotation of the method + * @param enteredState the state which is entered + * @return true if the method has to be invoked on this state enter + */ + private fun shouldInvokeMethod(at: StateEnter, enteredState: State): Boolean { + return State(at.state).equals(enteredState) + } + + /** + * Helper method for StateLeave annotation + * @param at the annotation of the method + * @param enteredState the state which is entered + * @return true if the method has to be invoked on this state leave + */ + private fun shouldInvokeMethod(at: StateLeave, enteredState: State): Boolean { + return State(at.state).equals(enteredState) + } + + /** + * Returns the names of parameters of a method which is annotated with [StateTransition], [org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker.StateEnter] or [org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker.StateLeave] + * @param method the method which is checked + * @return List of parameter names + */ + private fun getMethodRelatedParams(method: Method): List { + if (method.isAnnotationPresent(StateLeave::class.java)) { + val at = method.getAnnotation( + StateLeave::class.java + ) + return Arrays.stream(at.data).toList() + } else if (method.isAnnotationPresent(StateTransition::class.java)) { + val at = method.getAnnotation( + StateTransition::class.java + ) + return Arrays.stream(at.data).toList() + } else if (method.isAnnotationPresent(StateEnter::class.java)) { + val at = method.getAnnotation( + StateEnter::class.java + ) + return Arrays.stream(at.data).toList() + } + throw StateProviderException("The method " + method.name + " is not a state provider method") + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/abstractworker/StateProviderException.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/abstractworker/StateProviderException.kt new file mode 100644 index 000000000..9deb16b1e --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/abstractworker/StateProviderException.kt @@ -0,0 +1,20 @@ +package org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker + +import java.io.Serial + +/** + * Exception thrown by the [StateProviderAnnotationParser] if an error occurs during parsing. + */ +@Suppress("unused") +class StateProviderException : RuntimeException { + constructor(message: String?) : super(message) + + constructor(message: String?, cause: Throwable?) : super(message, cause) + + constructor(cause: Throwable?) : super(cause) + + companion object { + @Serial + private val serialVersionUID = 2621424721657557641L + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/abstractworker/StateTransition.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/abstractworker/StateTransition.kt new file mode 100644 index 000000000..a49f7d283 --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/abstractworker/StateTransition.kt @@ -0,0 +1,13 @@ +package org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker + +/** + * With this annotation a method can be marked as a state transition method. + * The method will be called when a specific transition is triggered. + * (A transition is a state action pair) + * The method must have the same number of parameters as the data array. + * The parameters must be in the data container with the same key. + * `@StateTransition(state = "STATENAME", action = "ACTION", data = {"dataKey1", "dataKey2"} ` + */ +@Retention(AnnotationRetention.RUNTIME) +@Target(AnnotationTarget.FUNCTION, AnnotationTarget.PROPERTY_GETTER, AnnotationTarget.PROPERTY_SETTER) +annotation class StateTransition(val state: String, val action: String, val data: Array = []) \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/abstractworker/Worker.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/abstractworker/Worker.kt new file mode 100644 index 000000000..86c4a1fb5 --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/abstractworker/Worker.kt @@ -0,0 +1,226 @@ +package org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker + +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import org.vitrivr.engine.model3d.lwjglrender.util.fsm.controller.FiniteStateMachine +import org.vitrivr.engine.model3d.lwjglrender.util.fsm.controller.FiniteStateMachineException +import org.vitrivr.engine.model3d.lwjglrender.util.fsm.model.Action +import org.vitrivr.engine.model3d.lwjglrender.util.fsm.model.Graph +import org.vitrivr.engine.model3d.lwjglrender.util.fsm.model.State +import java.lang.reflect.InvocationTargetException +import java.util.concurrent.BlockingDeque + +/** + *

This Abstract Worker:

+ * Worker is the abstract class for the concrete worker thread. + * **The Abstract Worker provides:** + * + * * Loading the [Graph] from concrete worker implementation + * * Creating a finite state machine [FiniteStateMachine] + * * Waiting on a [org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker.Job] + * * Performing a [org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker.Job], by perform [Action] and walk with the [FiniteStateMachine] through the [Graph] + * * On each transition a [org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker.StateProviderAnnotationParser] to invoke the marked methods + * (with [org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker.StateEnter], [org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker.StateLeave], [org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker.StateTransition]) from the concrete worker implementation + * * Handling exceptions [StateProviderException] or [FiniteStateMachine] + * + * + * This abstract worker has to be extended by a **concrete worker** implementation. + * + * + *

Concrete worker that extends from abstract Worker

+ * The concrete worker has to implement all methods to do the concrete job `` + * If a method should be invoked on a state transition, the method has to be annotated with [org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker.StateEnter], [org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker.StateLeave], [org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker.StateTransition] + * The graph has to be generated on instantiation of concrete worker. It has to describe all legal transitions. + * Further it has to provide an initial state and a set of final states. + * + * + * On each legal transition the [org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker.StateProviderAnnotationParser] invokes the methods that are annotated with [org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker.StateEnter], [org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker.StateLeave], [org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker.StateTransition] + * The variant data is passed to the method as parameter, related to the defined key in the annotation. + * + * + * If a final state is reached, the job is finished and the worker waits for the next job. + * The concrete worker implementation has to handle the result of the job and return a [org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker.JobControlCommand] to the worker. + * + * + * If an exception is thrown, the worker calls the method [.onJobException] from the concrete worker implementation. + * The concrete worker implementation has to handle the exception and return a [org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker.JobControlCommand] to the worker. + * + * @param The type of the job + */ +@StateProvider +abstract class Worker( + /** + * The queue of jobs that are waiting to be performed. + * `` is the type of concrete implementation of [Job] + */ + private val jobs: BlockingDeque +) : + Runnable { + /** + * Flag to shut down the worker. + * After that the worker has to be reinitialized. + * Usually on shutdown of the application. + */ + private var shutdown = false + + /** + * The graph that describes the legal transitions for the concrete worker. + * The finite state machine will walk through this graph. + */ + private val graph: Graph + + /** + * The current job that is performed. + */ + protected var currentJob: T? = null + + /** + * Constructor for the abstract worker. + * Registers the queue of jobs that are waiting to be performed. + * Calls the abstract method [.createGraph], which has to be implemented by the concrete worker. + * + * @param jobs The queue of jobs that are waiting to be performed. + */ + init { + this.graph = this.createGraph() + } + + /** + * Abstract method to create the graph. + * The graph has to be generated on instantiation of concrete worker. + * It has to describe all legal transitions. + * Further it has to provide an initial state and a set of final states. + * + * @return The graph that describes the legal transitions for the concrete worker. + */ + protected abstract fun createGraph(): Graph + + /** + * Abstract method to handle the exception. + * The concrete worker implementation has to handle the exception and return a [org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker.JobControlCommand] to the worker. + * After that the job is finished and the worker waits for the next job. + * + * @param ex The exception that was thrown. + * @return The [JobControlCommand] to the worker. + */ + protected abstract fun onJobException(ex: Exception?): String? + + /** + * Worker thread loop. + */ + override fun run() { + this.shutdown = false + // While application is running + while (!this.shutdown) { + try { + //LOGGER.trace("Waiting for job. In Queue:" + this.jobs.size()); + LOGGER.debug( + "Perform job. In queue: {}", + jobs.size + ) + when (currentJob!!.type) { + JobType.ORDER -> this.performJob( + this.currentJob!! + ) + + JobType.CONTROL -> { + this.shutdown = true + LOGGER.info("Worker is shutting down.") + } + JobType.RESPONSE -> LOGGER.error("Worker response not handled.") + + } + } finally { + LOGGER.trace("Worker has performed Job. In Queue:" + jobs.size) + } + } + } + + /** + * Job loop. Perform a single Job + * + * 1. Setup the Statemachine with initialized graph + * 1. Gets the action sequence and the job data for this job + * 1. Do till final state in graph is reached (or exception is thrown) + * + * A single job loop: + * + * * Take action + * * Move to next state + * * The StateProviderAnnotationParser will call all methods in the Worker that were marked with a corresponding Annotation + * + * + * @param job Job to be performed. + * @see StateEnter + * + * @see StateLeave + * + * @see StateTransition + */ + fun performJob(job: T) { + this.currentJob = job + + // Mark if job is finished + var performed = false + // Setup the Statemachine with initialized graph + val fsm = FiniteStateMachine(this.graph) + + // Get the action queue + val actions = job!!.actions + // Get the job data + val data = job.data + + while (!performed) { + try { + // Take next action + val action = actions!!.take() + // S_{i}, The current state S_{i} is the state before the transition + val leavedState: State = fsm.currentState!! + // S_{i} -> T_{k} -> S_{i+1} Move to next state, get the transition + val currentTransition = fsm.moveNextState(action) + // S_{i+1}, The current state S_{i+1} is the state after the transition + val enteredState: State = fsm.currentState!! + // Instantiate the StateProviderAnnotationParser and run it. + val sap = StateProviderAnnotationParser() + sap.runTransitionMethods(this, leavedState, enteredState, currentTransition, data!!) + // Check if final state is reached + performed = graph.isFinalState(enteredState) + } catch (ex: FiniteStateMachineException) { + // Exception is thrown if an illegal transition is performed + LOGGER.error("Error in job transition. Abort: ", ex) + this.onJobException(ex) + performed = true + } catch (ex: InterruptedException) { + // Exception is thrown if the worker is interrupted (needed for blocking queue) + LOGGER.error("Critical interruption in job task. Abort: ", ex) + this.onJobException(ex) + performed = true + } catch (ex: InvocationTargetException) { + // This exception is thrown if an exception is thrown during invocation of the methods in the concrete worker + this.onJobException(ex) + LOGGER.error("Error in concrete wWorker. Abort: ", ex) + performed = true + } catch (ex: IllegalAccessException) { + this.onJobException(ex) + LOGGER.error("Error in concrete wWorker. Abort: ", ex) + performed = true + } finally { + } + } + } + + /** + * Method to put an action at the end of the action queue. + * Could be used to add an action to perform an error handling. + * + * @param action The action to be put at the end of the action queue. + */ + @Suppress("unused") + protected fun putActionFirst(action: Action?) { + currentJob!!.actions!!.addFirst(action) + } + + companion object { + private val LOGGER: Logger = LogManager.getLogger() + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/controller/FiniteStateMachine.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/controller/FiniteStateMachine.kt new file mode 100644 index 000000000..91b1e81a6 --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/controller/FiniteStateMachine.kt @@ -0,0 +1,81 @@ +package org.vitrivr.engine.model3d.lwjglrender.util.fsm.controller + +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import org.vitrivr.engine.model3d.lwjglrender.util.fsm.model.Action +import org.vitrivr.engine.model3d.lwjglrender.util.fsm.model.Graph +import org.vitrivr.engine.model3d.lwjglrender.util.fsm.model.State +import org.vitrivr.engine.model3d.lwjglrender.util.fsm.model.Transition + +/** + * Implements a FiniteStateMachine + * + * + * Inspired from Design pattern presented on Stackoverflow + * + * @see [ + * https://stackoverflow.com/questions/5923767/simple-state-machine-example-in-c/a> +](https://stackoverflow.com/questions/5923767/simple-state-machine-example-in-c) */ +class FiniteStateMachine( + /** + * Hashtable for Unique State transitions + */ + private val graph: Graph +) { + var LOGGER: Logger = LogManager.getLogger(FiniteStateMachine::class.java) + /** + * @return current State of FSM + */ + /** + * The current State + */ + var currentState: State? + private set + + /** + * Constructs a [FiniteStateMachine] + * Sets the initial state + * @param graph the graph which contains all states and transitions + * and the initial state + */ + init { + this.currentState = graph.initialState() + } + + + /** + * Gives a preview on next state with a hypothetical command + * + * @param command given hypothetical command + * @return NextState resulting State + * @throws FiniteStateMachineException if transition is not valid + */ + @Throws(FiniteStateMachineException::class) + fun previewNextState(command: Action): State? { + val transition = Transition( + currentState!!, command + ) + if (graph.containsTransition(transition)) { + return graph.getNextState(transition) + } else { + LOGGER.error("FSM transition to next state failed!") + throw FiniteStateMachineException("Invalid transition: " + currentState.toString() + " -> " + command.toString()) + } + } + + /** + * Moves the FSM to the next state + * + * @param command given command + * @return the resulting state after transition + * @throws FiniteStateMachineException if transition is not valid + */ + @Throws(FiniteStateMachineException::class) + fun moveNextState(command: Action): Transition { + val performedTransition = Transition( + currentState!!, command + ) + this.currentState = this.previewNextState(command) + return performedTransition + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/controller/FiniteStateMachineException.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/controller/FiniteStateMachineException.kt new file mode 100644 index 000000000..dce571638 --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/controller/FiniteStateMachineException.kt @@ -0,0 +1,20 @@ +package org.vitrivr.engine.model3d.lwjglrender.util.fsm.controller + +import java.io.Serial + +/** + * Exception thrown by the FiniteStateMachine if an illegal state transition is attempted. + */ +@Suppress("unused") +class FiniteStateMachineException : Exception { + constructor(message: String?) : super(message) + + constructor(message: String?, cause: Throwable?) : super(message, cause) + + constructor(cause: Throwable?) : super(cause) + + companion object { + @Serial + private val serialVersionUID = 500983167704077039L + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/model/Action.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/model/Action.kt new file mode 100644 index 000000000..c1109fb25 --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/model/Action.kt @@ -0,0 +1,88 @@ +package org.vitrivr.engine.model3d.lwjglrender.util.fsm.model + +import org.vitrivr.engine.model3d.lwjglrender.util.datatype.Variant + +/** + * Represents an Action in a Finite State Machine + * The Action is uniquely identified by its name + * The name is used to create a hashcode and to compare actions + * The Action can contain data + * The data provides in an action is only visible to the state transition which were triggered by the action + */ +class Action { + /** + * Unique Name of the action + */ + private val name: String + + /** + * @return the data of the action + */ + /** + * Data of the action + */ + val data: Variant? + + /** + * Creates a new Action with a unique name + * + * @param name of the action, should be unique + */ + constructor(name: String) { + this.name = name + this.data = null + } + + /** + * Creates a new Action with a unique name and additional data + * + * @param name of the action, should be unique + * @param data of the action + */ + constructor(name: String, data: Variant?) { + this.name = name + this.data = data + } + + /** + * @return true if the action has data + */ + @Suppress("unused") + fun hasData(): Boolean { + return this.data != null + } + + /** + * Creates a Unique hashcode for command + * + * @return a hashCode for Command + */ + override fun hashCode(): Int { + return 17 + 31 * name.hashCode() + } + + /** + * Implements the equals method for state Transition + * + * @param obj to compare + * @return true if the states are equal + */ + override fun equals(obj: Any?): Boolean { + if (this === obj) { + return true + } + if (obj !is Action) { + return false + } + return this.name == obj.name + } + + /** + * Returns the unique name of the action + * + * @return name of the action + */ + override fun toString(): String { + return this.name + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/model/Graph.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/model/Graph.kt new file mode 100644 index 000000000..99b7d1e57 --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/model/Graph.kt @@ -0,0 +1,132 @@ +package org.vitrivr.engine.model3d.lwjglrender.util.fsm.model + +import java.io.File +import java.io.IOException +import java.nio.file.Files +import java.util.* +import kotlin.collections.HashMap + +/** + * Represents a graph that can be used by an FSM to traverse. + */ +class Graph @JvmOverloads constructor( + /** + * The transitions of the graph. + */ + private val transitions: HashMap, + /** + * The initial state of the graph. + */ + private val initialState: State, goalStates: HashSet, + export: Boolean = false +) { + /** + * The set of goal states of the graph.* + */ + private val goalStates: Set = + goalStates + + /** + * @see Graph.Graph + * @param export parameter to disable the export of the graph. + */ + /** + * Creates a new Graph. + * The setup process can be as follows: + * The transitions have to describe for each state which action leads to which state. + * + * + * `return new Graph(new Hashtable<>() {{ + * {put(new Transition(new State("Startup"), new Action("wait")), new State("Startup"));} + * {put(new Transition(new State("Startup"), new Action("print")), new State("Print"));} + * ... + * }}, new State("Startup"), new HashSet<>() {{ + * {add(new State("Result"));} + * }});` + * + * @param transitions The transitions of the graph as a hashtable. + * @param initialState The initial state of the graph. + * @param goalStates The set of goal states of the graph. + */ + init { + if (export) { + this.export() + } + } + + /** + * Returns the initial state of the graph. + * + * @return The initial state of the graph. + */ + fun initialState(): State { + return this.initialState + } + + /** + * True if graph contains the transition + * @param transition transition to check + * @return true if graph contains the transition + */ + fun containsTransition(transition: Transition): Boolean { + return transitions.containsKey(transition) + } + + /** + * Returns the next state based on given transition which is a unique state action pair. + * @param transition The transition to check. + * @return The next state for a given transition. + */ + fun getNextState(transition: Transition): State? { + return transitions[transition] + } + + /** + * Returns true if the given state is a goal state. + * @param enteredState The state to check. + * @return True if the given state is a goal state. + */ + fun isFinalState(enteredState: State): Boolean { + return goalStates.contains(enteredState) + } + + /** + * Generates a graph viz string representation of the graph. + */ + fun toString(flavour: String?): String { + val sb = StringBuilder() + sb.append("@startuml") + sb.append("\n") + sb.append("[*] --> ") + sb.append(this.initialState) + sb.append("\n") + for ((key, value) in this.transitions) { + sb.append(key.state.toString()) + sb.append(" --> ") + sb.append(value.toString()) + sb.append(" : ") + sb.append(key.command.toString()) + sb.append("\n") + } + for (entry in this.goalStates) { + sb.append(entry.toString()) + sb.append(" --> [*]") + sb.append("\n") + } + sb.append("@enduml") + + return sb.toString() + } + + /** + * Helper for exports the graph to a file. + */ + private fun export() { + val file = File("fsm.txt") + try { + Files.writeString(file.toPath(), this.toString("plantuml")) + } catch (e: IOException) { + e.printStackTrace() + } + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/model/State.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/model/State.kt new file mode 100644 index 000000000..598ba87ee --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/model/State.kt @@ -0,0 +1,52 @@ +package org.vitrivr.engine.model3d.lwjglrender.util.fsm.model + +/** + * Represents a State in a Finite State Machine + * The State is uniquely identified by its name + * The name is used to create a hashcode and to compare states + */ +class State +/** + * Creates a new State with a unique name + * + * @param name of the state, should be unique + */( + /** + * Unique Name of the state + */ + private val name: String +) { + /** + * Creates a Unique hashcode for current state + * + * @return a hashCode for state + */ + override fun hashCode(): Int { + return 17 + 31 * name.hashCode() + } + + /** + * Implements the equals method for state + * + * @param obj to compare + * @return true if the states are equal + */ + override fun equals(obj: Any?): Boolean { + if (this === obj) { + return true + } + if ((obj !is State)) { + return false + } + return this.name == obj.name + } + + /** + * Returns the unique name of the state + * + * @return name of the state + */ + override fun toString(): String { + return this.name + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/model/Transition.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/model/Transition.kt new file mode 100644 index 000000000..91b78660c --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/fsm/model/Transition.kt @@ -0,0 +1,59 @@ +package org.vitrivr.engine.model3d.lwjglrender.util.fsm.model + +/** + * StateTransition describes a transition out of a currentState with a command Therefor it consists + * of a state action pair The state action pair creates a unique hashcode + */ +class Transition +/** + * Constructor for StateTransition + * + * @param state (current / outgoing) state of the transition + * @param command command which triggers the transition from state + */ +( + /** (current / outgoing) state of the transition */ + val state: State, + /** command which triggers the transition from state */ + val command: Action +) { + /** + * Returns the (current / outgoing) state of the transition + * + * @return current state + */ + /** + * Returns the command of the transition + * + * @return command + */ + + /** + * Creates a Unique hashcode for combination current state and command + * + * @return a hashCode for StateTransition + */ + override fun hashCode(): Int { + return 17 + 31 * state.hashCode() + 31 * command.hashCode() + } + + /** + * Implements the equals method for state Transition + * + * @param obj to compare + * @return true if equal + */ + override fun equals(obj: Any?): Boolean { + if (obj === this) { + return true + } + if (obj !is Transition) { + return false + } + return state.equals(obj.state) && command.equals(obj.command) + } + + override fun toString(): String { + return "Transition{state=$state, command=$command}" + } +} diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/math/MathConstants.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/math/MathConstants.kt new file mode 100644 index 000000000..14ad6a277 --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/math/MathConstants.kt @@ -0,0 +1,74 @@ +package org.vitrivr.engine.model3d.lwjglrender.util.math + +import kotlin.math.sqrt + + +object MathConstants { + /** + * Definition of the golden ratio PHI. + */ + val PHI: Double = ((1.0 + sqrt(5.0)) / 2.0) + + /** + * Square-root of three. + */ + val SQRT3: Double = sqrt(3.0) + + /** + * Square-root of two. + */ + val SQRT2: Double = sqrt(2.0) + + /** + * Square-root of two. + */ + val SQRT1_5: Double = sqrt(1.5) + + + /** + * Defines the vertices of a regular Cube. + */ + val VERTICES_3D_CUBE: Array = arrayOf( + doubleArrayOf(1.0, 1.0, 1.0), + doubleArrayOf(-1.0, -1.0, -1.0), + doubleArrayOf(1.0, -1.0, -1.0), + doubleArrayOf(-1.0, -1.0, 1.0), + doubleArrayOf(-1.0, 1.0, -1.0), + doubleArrayOf(-1.0, 1.0, 1.0), + doubleArrayOf(1.0, -1.0, 1.0), + doubleArrayOf(1.0, 1.0, -1.0) + ) + + val VERTICES_3D_3TRIANGLES: Array = arrayOf( + doubleArrayOf(0.0, 0.0, SQRT3), doubleArrayOf(SQRT1_5, 0.0, -SQRT1_5), doubleArrayOf(-SQRT1_5, 0.0, -SQRT1_5), + doubleArrayOf(-1.0, 1.0, 1.0), doubleArrayOf(-1.0, 1.0, -1.0), doubleArrayOf(-SQRT1_5, SQRT1_5, 0.0), + doubleArrayOf(1.0, -1.0, 1.0), doubleArrayOf(-SQRT1_5, -SQRT1_5, 0.0), doubleArrayOf(1.0, -1.0, -1.0), + ) + + + /** + * Defines the vertices of a regular Dodecahedron. + */ + val VERTICES_3D_DODECAHEDRON: Array = arrayOf( + doubleArrayOf(1.0, 1.0, 1.0), + doubleArrayOf(-1.0, -1.0, -1.0), + doubleArrayOf(1.0, -1.0, -1.0), + doubleArrayOf(-1.0, -1.0, 1.0), + doubleArrayOf(-1.0, 1.0, -1.0), + doubleArrayOf(-1.0, 1.0, 1.0), + doubleArrayOf(1.0, -1.0, 1.0), + doubleArrayOf(1.0, 1.0, -1.0), + doubleArrayOf(0.0, 1 / PHI, PHI), + doubleArrayOf(0.0, -1 / PHI, PHI), + doubleArrayOf(0.0, 1 / PHI, -PHI), + doubleArrayOf(0.0, -1 / PHI, -PHI), + doubleArrayOf(1 / PHI, PHI, 0.0), + doubleArrayOf(-1 / PHI, PHI, 0.0), + doubleArrayOf(1 / PHI, -PHI, 0.0), + doubleArrayOf(-1 / PHI, -PHI, 0.0), + doubleArrayOf(PHI, 0.0, 1 / PHI), + doubleArrayOf(-PHI, 0.0, 1 / PHI), + doubleArrayOf(PHI, 0.0, -1 / PHI), + doubleArrayOf(-PHI, 0.0, -1 / PHI) + ) +} \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/texturemodel/entroopyoptimizer/ModelEntropyOptimizer.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/texturemodel/entroopyoptimizer/ModelEntropyOptimizer.kt new file mode 100644 index 000000000..6d1c7c494 --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/texturemodel/entroopyoptimizer/ModelEntropyOptimizer.kt @@ -0,0 +1,328 @@ +package org.vitrivr.engine.model3d.lwjglrender.util.texturemodel.entroopyoptimizer + +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import org.joml.Vector3f +import org.vitrivr.engine.core.model.mesh.texturemodel.IModel +import org.vitrivr.engine.core.model.mesh.texturemodel.util.entropyoptimizer.EntopyCalculationMethod +import org.vitrivr.engine.core.model.mesh.texturemodel.util.entropyoptimizer.EntropyOptimizerStrategy +import org.vitrivr.engine.core.model.mesh.texturemodel.util.entropyoptimizer.OptimizerOptions +import java.util.stream.IntStream +import kotlin.math.abs +import kotlin.math.ln +import kotlin.math.log10 +import kotlin.math.max + +/** + * Static class for optimizing the view vector of a model to maximize the viewpoint entropy on the model. + */ +object ModelEntropyOptimizer { + private val LOGGER: Logger = LogManager.getLogger() + + /** + * Calculates the view vector with the maximum entropy of the model. Uses standard Options for optimizer and entropy calculation. + * + * @param model Model to calculate the view vector for. + * @return View vector with the maximum entropy. + */ + fun getViewVectorWithMaximizedEntropy(model: IModel): Vector3f { + return getViewVectorWithMaximizedEntropy(model, OptimizerOptions()) + } + + /** + * Calculates the view vector with the maximum entropy of the model. + * + * @param model Model to calculate the view vector for. + * @param options Options for the optimizer and entropy calculation. + * @return View vector with the maximum entropy. + */ + fun getViewVectorWithMaximizedEntropy(model: IModel, options: OptimizerOptions): Vector3f { + val normals = model.getAllNormals() + val viewVector = options.initialViewVector + + val list: MutableList = List(normals.size) { Vector3f() }.toMutableList() + for ((idx, i) in normals.withIndex()){ + val vector3f = Vector3f(i.x, i.y, i.z) + list[idx] = vector3f + } + val maxEntropyViewVector = optimize(options, list, Vector3f(viewVector.x, viewVector.y, viewVector.z)) + return maxEntropyViewVector + } + + /** + * Wrapper for the optimizer strategy. Optimizes the view vector for the given model with the chosen EntropyOptimizer Strategy. + * + * @param options Options for the optimizer and entropy calculation. + * @param normals List of normals of the model. + * @param viewVector Initial view vector. + * @return Optimized view vector. + */ + private fun optimize(options: OptimizerOptions, normals: List, viewVector: Vector3f): Vector3f { + val optimizer = options.optimizer + return when (optimizer) { + EntropyOptimizerStrategy.RANDOMIZED -> { + optimizeRandomized(options, normals, viewVector) + } + + EntropyOptimizerStrategy.NEIGHBORHOOD -> { + optimizeNeighborhood(options, normals, viewVector) + } + + else -> { + Vector3f(0f, 0f, 1f) + } + } + } + + /** + * Optimizes the view vector for the given model with the randomized EntropyOptimizer Strategy. + * + * @param options Options for the optimizer and entropy calculation. + * + * * Uses the option iterations. For each iteration a random view vector is generated. + * * Uses the option zoomOutFactor. The view vector is zoomed out by this factor. + * + * @param normals List of normals of the model. + * @param viewVector Initial view vector. + * @return Optimized view vector. + */ + private fun optimizeRandomized(options: OptimizerOptions, normals: List, viewVector: Vector3f): Vector3f { + val t0 = System.currentTimeMillis() + val iterations = options.iterations + var maxEntropy = calculateEntropy(options, normals, viewVector) + var maxEntropyViewVector = viewVector + var ic = 0 + ic = 0 + while (ic < iterations) { + val randomViewVector = Vector3f( + (Math.random() - 0.5).toFloat() * 2f, (Math.random() - 0.5).toFloat() * 2f, + (Math.random() - 0.5).toFloat() * 2f + ) + randomViewVector.normalize() + // For Entropy calculation benchmarking comment out the following lines. (Logger is slow) + // var t0_0 = System.nanoTime(); + val entropy = calculateEntropy(options, normals, randomViewVector) + // var t1_0 = System.nanoTime(); + // LOGGER.trace("Entropy: {} for ViewVector: {} took {} ns", entropy, randomViewVector, t1_0 - t0_0); + if (entropy > maxEntropy) { + maxEntropy = entropy + maxEntropyViewVector = randomViewVector + } + ic++ + } + val t1 = System.currentTimeMillis() + LOGGER.trace( + "Optimization took {} ms with {} iterations for {} normals, getting a max. Entropy of {}. Resulting in {} us/normal", + t1 - t0, ic + 1, normals.size, maxEntropy, (t1 - t0) * 1000L / normals.size.toLong() + ) + return maxEntropyViewVector.mul(options.zoomOutFactor) + } + + /** + * Optimizes the view vector for the given model with the neighborhood EntropyOptimizer Strategy. + * + * @param options Options for the optimizer and entropy calculation. + * @param normals List of normals of the model. + * @param viewVector Initial view vector. + * @return Optimized view vector. + */ + private fun optimizeNeighborhood( + options: OptimizerOptions, + normals: List, + viewVector: Vector3f + ): Vector3f { + return Vector3f(0f, 0f, 1f) + } + + /** + * Wrapper for the entropy calculation strategy. Calculates the entropy of the model for the given view vector. + * + * @param options Options for the optimizer and entropy calculation. + * @param normals List of normals of the model. + * @param viewVector View vector. + * @return Entropy of the model for the given view vector. + */ + private fun calculateEntropy(options: OptimizerOptions, normals: List, viewVector: Vector3f): Float { + val method = options.method + return when (method) { + EntopyCalculationMethod.RELATIVE_TO_TOTAL_AREA -> { + calculateEntropyRelativeToTotalArea(normals, viewVector) + } + + EntopyCalculationMethod.RELATIVE_TO_TOTAL_AREA_WEIGHTED -> { + calculateEntropyRelativeToTotalAreaWeighted(normals, viewVector, options) + } + + EntopyCalculationMethod.RELATIVE_TO_PROJECTED_AREA -> { + calculateEntropyRelativeToProjectedArea(normals, viewVector) + } + + else -> { + 0f + } + } + } + + /** + * Calculates the entropy of the model for the given view vector relative to the projected area of the model. + * + * @param normals List of normals of the model. + * @param viewVector View vector. + * @return Entropy of the model for the given view vector. + */ + private fun calculateEntropyRelativeToProjectedArea(normals: List, viewVector: Vector3f): Float { + return 0f + } + + /** + * Calculates the entropy of the model for the given view vector relative to the projected area of the model. see: [Google Scholar](https://scholar.google.ch/scholar?hl=de&as_sdt=0%2C5&as_vis=1&q=Viewpoint+selection+using+viewpoint+entrop&btnG=) see: href="https://citeseerx.ist.psu.edu/document?repid=rep1&type=pdf&doi=b854422671e5469373fd49fb3a916910b49a6920">Paper + * + * @param normals List of normals of the model. + * @param viewVector View vector. + * @return Entropy of the model for the given view vector. + */ + private fun calculateEntropyRelativeToTotalArea_old(normals: List, viewVector: Vector3f): Float { + val areas = java.util.ArrayList(normals.size) + val projected = java.util.ArrayList(normals.size) + normals.stream().map { normal: Vector3f? -> + viewVector.dot( + normal + ) / 2f + }.forEach { e: Float -> areas.add(e) } + areas.stream().map { area: Float -> if (area > 0f) area else 0f }.forEach { e: Float -> + projected.add( + e + ) + } + val totalArea = areas.stream().map { a: Float? -> + abs( + a!! + ) + }.reduce( + 0f + ) { a: Float, b: Float -> java.lang.Float.sum(a, b) } + val relativeProjected = projected.stream().map { x: Float -> x / totalArea }.toList() + val logRelativeProjected = relativeProjected.stream().map { x: Float? -> + log2( + x!! + ) + }.toList() + assert(relativeProjected.size == logRelativeProjected.size) + val result = IntStream.range(0, relativeProjected.size) + .mapToObj { ic: Int -> + relativeProjected[ic] * logRelativeProjected[ic] + } + .reduce(0f) { a: Float, b: Float -> java.lang.Float.sum(a, b) } + val entropy = -result + return entropy + } + + /** + * Calculates the entropy of the model for the given view vector relative to the projected area of the model. see: [Google Scholar](https://scholar.google.ch/scholar?hl=de&as_sdt=0%2C5&as_vis=1&q=Viewpoint+selection+using+viewpoint+entrop&btnG=) see: href="https://citeseerx.ist.psu.edu/document?repid=rep1&type=pdf&doi=b854422671e5469373fd49fb3a916910b49a6920">Paper + * + * @param normals List of normals of the model. + * @param viewVector View vector. + * @return Entropy of the model for the given view vector. + */ + private fun calculateEntropyRelativeToTotalArea2(normals: List, viewVector: Vector3f): Float { + var areas: FloatArray? = FloatArray(normals.size) + val projected = FloatArray(normals.size) + var totalArea = 0f + for (ic in normals.indices) { + areas!![ic] = viewVector.dot(normals[ic]) / 2f + projected[ic] = max(areas[ic].toDouble(), 0.0).toFloat() + totalArea += abs(areas[ic].toDouble()).toFloat() + } + areas = null + var entropy = 0f + for (ic in normals.indices) { + projected[ic] /= totalArea + entropy += projected[ic] * log2( + projected[ic] + ) + } + + return -entropy + } + + /** + * Calculates the entropy of the model for the given view vector relative to the projected area of the model. see: [Google Scholar](https://scholar.google.ch/scholar?hl=de&as_sdt=0%2C5&as_vis=1&q=Viewpoint+selection+using+viewpoint+entrop&btnG=) see: href="https://citeseerx.ist.psu.edu/document?repid=rep1&type=pdf&doi=b854422671e5469373fd49fb3a916910b49a6920">Paper + * + * @param normals List of normals of the model. + * @param viewVector View vector. + * @param opts Optimizer options. + * @return Entropy of the model for the given view vector. + */ + private fun calculateEntropyRelativeToTotalAreaWeighted( + normals: List, + viewVector: Vector3f, + opts: OptimizerOptions + ): Float { + val weightedNormals = ArrayList(normals.size) + normals.stream().parallel().forEach { n: Vector3f -> + if (n.y > 0) { + weightedNormals.add(Vector3f(n.x, n.y * opts.yPosWeight, n.z)) + } else { + weightedNormals.add(Vector3f(n.x, n.y * opts.yNegWeight, n.z)) + } + } + return calculateEntropyRelativeToTotalArea(weightedNormals, viewVector) + } + + /** + * Calculates the entropy of the model for the given view vector relative to the projected area of the model. see: [Google Scholar](https://scholar.google.ch/scholar?hl=de&as_sdt=0%2C5&as_vis=1&q=Viewpoint+selection+using+viewpoint+entrop&btnG=) see: href="https://citeseerx.ist.psu.edu/document?repid=rep1&type=pdf&doi=b854422671e5469373fd49fb3a916910b49a6920">Paper + * + * @param normals List of normals of the model. + * @param viewVector View vector. + * @return Entropy of the model for the given view vector. + */ + private fun calculateEntropyRelativeToTotalArea(normals: List, viewVector: Vector3f): Float { + val areas = FloatArray(normals.size) + val projected = FloatArray(normals.size) + var totalArea = 0f + IntStream.range(0, normals.size).parallel().forEach { ic: Int -> + areas[ic] = viewVector.dot(normals[ic]) + projected[ic] = max(areas[ic].toDouble(), 0.0).toFloat() + areas[ic] = abs(areas[ic].toDouble()).toFloat() + } + + for (ic in normals.indices) { + totalArea += normals[ic].length() + } + + var entropy = 0f + val finalTotalArea = totalArea + IntStream.range(0, normals.size).parallel().forEach { ic: Int -> + projected[ic] /= finalTotalArea + projected[ic] = + projected[ic] * log2( + projected[ic] + ) + } + + for (ic in normals.indices) { + entropy += projected[ic] + } + + return -entropy + } + + + /** + * Static values for log base 2, due to performance reasons. + */ + private val LOG10OF2 = log10(2.0).toFloat() + + /** + * Calculates the logarithm of a number to base 2. If x is 0, 0 is returned. + * + * @param x The number to calculate the logarithm for. + * @return log2(x) + */ + private fun log2(x: Float): Float { + if (x <= 0f) { + return 0f + } + return (ln(x.toDouble()).toFloat() / LOG10OF2) + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/texturemodel/viewpoint/ViewpointStrategy.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/texturemodel/viewpoint/ViewpointStrategy.kt new file mode 100644 index 000000000..f65ab163d --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/util/texturemodel/viewpoint/ViewpointStrategy.kt @@ -0,0 +1,51 @@ +package org.vitrivr.engine.model3d.lwjglrender.util.texturemodel.viewpoint + +/** + * For benchmark purposes. Since the Extractor does not support options, strategy should be implemented in a static way + */ +enum class ViewpointStrategy { + /** + * Randomly selects a viewpoint + */ + RANDOM, + + /** + * Selects the viewpoint from the front (0,0,1) + */ + FRONT, + + /** + * Selects the viewpoint from the upper left (-1,1,1) + */ + UPPER_LEFT, + + /** + * Runs the viewpoint entropy maximization algorithm to find the best viewpoint + */ + VIEWPOINT_ENTROPY_MAXIMIZATION_RANDOMIZED, + + /** + * Runs the viewpoint entropy maximization algorithm with y plane attraction to find the best viewpoint + */ + VIEWPOINT_ENTROPY_MAXIMIZATION_RANDOMIZED_WEIGHTED, + + /** + * Takes multiple images and aggregates the vectors using k-means + */ + MULTI_IMAGE_KMEANS, + + /** + * Takes multiple images and aggregates the vectors using the projected mean + */ + MULTI_IMAGE_PROJECTEDMEAN, + + /** + * Takes multiple images and embeds them as a video + */ + MULTI_IMAGE_FRAME, + + /** + * Creates a 2x2 image + */ + MULTI_IMAGE_2_2, +} \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/window/Window.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/window/Window.kt new file mode 100644 index 000000000..60826b6cb --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/window/Window.kt @@ -0,0 +1,211 @@ +package org.vitrivr.engine.model3d.lwjglrender.window + +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import org.lwjgl.glfw.* +import org.lwjgl.opengl.GL30 +import org.lwjgl.system.MemoryUtil +import java.util.concurrent.Callable + +/** + * This class represents a window that can be used for rendering. + * It is based on the LWJGL3 library. + * It can be used for both headless and visible rendering. + */ +class Window( + title: String?, opts: WindowOptions, + /** + * The function that is called when the window is resized. + */ + private val resizeFunc: Callable +) { + /** + * The handle to the window. + */ + private val windowHandle: Long + /** + * Returns the height of the window. + * + * @return Height of the window. + */ + /** + * The height of the window. + */ + var height: Int + private set + /** + * Returns the width of the window. + * + * @return Width of the window. + */ + /** + * The width of the window. + */ + var width: Int + private set + + /** + * Constructor for Window. + * + * @param title Title of the window. + * @param opts Options for the window. + * @param resizeFunc Function that is called when the window is resized. + */ + init { + LOGGER.trace("Try creating window '{}'...", title) + check(GLFW.glfwInit()) { "Unable to initialize GLFW" } + LOGGER.trace("GLFW initialized") + + GLFW.glfwDefaultWindowHints() + // Window should be invisible for basic rendering + GLFW.glfwWindowHint(GLFW.GLFW_VISIBLE, GL30.GL_FALSE) + // Setting for headless rendering with MESA and Xvfb + // See: https://github.com/vitrivr/cineast/blob/e5587fce1b5675ca9f6dbbfd5c17eb1880a98ce3/README.md + //GLFW.glfwWindowHint(GLFW.GLFW_CONTEXT_CREATION_API, GLFW.GLFW_OSMESA_CONTEXT_API); + // Switch off resize callback + GLFW.glfwWindowHint(GLFW.GLFW_RESIZABLE, GL30.GL_FALSE) + + // Sets the OpenGL version number to MAJOR.MINOR e.g. 3.2 + GLFW.glfwWindowHint(GLFW.GLFW_CONTEXT_VERSION_MAJOR, 3) + GLFW.glfwWindowHint(GLFW.GLFW_CONTEXT_VERSION_MINOR, 2) + + // Depending on the Options, the OpenGL profile can be set to either CORE or COMPAT (or ANY) + // GLFW_OPENGL_COMPAT_PROFILE keeps the outdated functionality + // GLFW_OPENGL_CORE_PROFILE removes the deprecated functionality + // GLFW_OPENGL_ANY_PROFILE is used for version 3.2 and below + if (opts.compatibleProfile) { + GLFW.glfwWindowHint(GLFW.GLFW_OPENGL_PROFILE, GLFW.GLFW_OPENGL_COMPAT_PROFILE) + } else { + GLFW.glfwWindowHint(GLFW.GLFW_OPENGL_PROFILE, GLFW.GLFW_OPENGL_CORE_PROFILE) + GLFW.glfwWindowHint(GLFW.GLFW_OPENGL_FORWARD_COMPAT, GL30.GL_TRUE) + } + + // Set window size if set in options, otherwise use maximum size of primary monitor + if (opts.width > 0 && opts.height > 0) { + this.width = opts.width + this.height = opts.height + } else { + GLFW.glfwWindowHint(GLFW.GLFW_MAXIMIZED, GLFW.GLFW_TRUE) + val vidMode = checkNotNull(GLFW.glfwGetVideoMode(GLFW.glfwGetPrimaryMonitor())) + this.width = vidMode.width() + this.height = vidMode.height() + } + + LOGGER.trace( + "Try creating window '{}' with size {}x{}...", title, + this.width, + this.height + ) + this.windowHandle = GLFW.glfwCreateWindow(this.width, this.height, title, MemoryUtil.NULL, MemoryUtil.NULL) + if (this.windowHandle == MemoryUtil.NULL) { + throw RuntimeException("Failed to create the GLFW window") + } + + // Setup the callbacks for the glfw window. + // Resize and key Callback are not used for headless rendering. + val resizeCallback = GLFW.glfwSetFramebufferSizeCallback( + this.windowHandle + ) { window: Long, w: Int, h: Int -> + this.resized( + w, + h + ) + } + + val errorCallback = GLFW.glfwSetErrorCallback { errorCode: Int, msgPtr: Long -> + LOGGER.error( + "Error code [{}], msg [{}]", + errorCode, + MemoryUtil.memUTF8(msgPtr) + ) + } + + val keyCallback = GLFW.glfwSetKeyCallback( + this.windowHandle + ) { window: Long, key: Int, scancode: Int, action: Int, mods: Int -> + if (key == GLFW.GLFW_KEY_ESCAPE && action == GLFW.GLFW_RELEASE) { + GLFW.glfwSetWindowShouldClose(window, true) // We will detect this in the rendering loop + } + } + + GLFW.glfwMakeContextCurrent(this.windowHandle) + + if (opts.fps > 0) { + GLFW.glfwSwapInterval(0) + } else { + GLFW.glfwSwapInterval(1) + } + + // Set the window to be visible if not headless rendering + if (!opts.hideWindow) { + GLFW.glfwShowWindow(this.windowHandle) + } + + val arrWidth = IntArray(1) + val arrHeight = IntArray(1) + GLFW.glfwGetFramebufferSize(this.windowHandle, arrWidth, arrHeight) + this.width = arrWidth[0] + this.height = arrHeight[0] + } + + /** + * Removes all callbacks and destroys the window. + */ + fun cleanup() { + Callbacks.glfwFreeCallbacks(this.windowHandle) + GLFW.glfwDestroyWindow(this.windowHandle) + GLFW.glfwTerminate() + val callback = GLFW.glfwSetErrorCallback(null) + callback?.free() + } + + /** + * Checks if a key is pressed. + * @param keyCode Key code to check. + * @return True if key is pressed, false otherwise. + */ + fun isKeyPressed(keyCode: Int): Boolean { + return GLFW.glfwGetKey(this.windowHandle, keyCode) == GLFW.GLFW_PRESS + } + + /** + * polls all pending events. + */ + fun pollEvents() { + GLFW.glfwPollEvents() + } + + /** + * Callback for window resize. + * + * @param width New width of the window. + * @param height New height of the window. + */ + protected fun resized(width: Int, height: Int) { + this.width = width + this.height = height + try { + resizeFunc.call() + } catch (ex: Exception) { + LOGGER.error("Error calling resize callback", ex) + } + } + + /** + * Updates the window. + */ + fun update() { + GLFW.glfwSwapBuffers(this.windowHandle) + } + + /** + * Indicates if the window should be closed. + */ + fun windowShouldClose(): Boolean { + return GLFW.glfwWindowShouldClose(this.windowHandle) + } + + companion object { + private val LOGGER: Logger = LogManager.getLogger() + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/window/WindowOptions.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/window/WindowOptions.kt new file mode 100644 index 000000000..4f450532b --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/lwjglrender/window/WindowOptions.kt @@ -0,0 +1,55 @@ +package org.vitrivr.engine.model3d.lwjglrender.window + +import java.io.Serializable + + +/** + * This class holds all options for a window. + */ +open class WindowOptions : Serializable { + /** + * set to true if the window should be created with a compatible profile + */ + var compatibleProfile: Boolean = false + + /** + * Frames per second. If set to -1, the fps is unlimited. + */ + var fps: Int = -1 + + /** + * Updates per second. If set to -1, the ups is unlimited. + */ + var ups: Int = 30 + + /** + * The height of the window. + */ + var height: Int = 400 + + /** + * The width of the window. + */ + var width: Int = 400 + + /** + * Hide the window after creation. + */ + var hideWindow: Boolean = true + + /** + * Empty constructor for WindowOptions. + */ + constructor() + + /** + * Basic Constructor for WindowOptions. + * + * @param width Width of the window. + * @param height Height of the window. + */ + constructor(width: Int, height: Int) { + this.width = width + this.height = height + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/model/voxel/VoxelModel.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/model/voxel/VoxelModel.kt index c34627562..925c158f3 100644 --- a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/model/voxel/VoxelModel.kt +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/model/voxel/VoxelModel.kt @@ -119,7 +119,7 @@ data class VoxelModel( * @param x x position of the Voxel. * @param y y position of the Voxel. * @param z z position of the Voxel. - * @return Vector3f containing the center of the voxel. + * @return org.vitrivr.engine.core.model.types.Vector3f containing the center of the voxel. * @throws ArrayIndexOutOfBoundsException If one of the three indices is larger than the grid. */ fun getVoxelCenter(x: Int, y: Int, z: Int): Vector3f { diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/model/voxel/Voxelizer.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/model/voxel/Voxelizer.kt index 498191f3b..6ffa20e91 100644 --- a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/model/voxel/Voxelizer.kt +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/model/voxel/Voxelizer.kt @@ -5,8 +5,9 @@ import org.apache.logging.log4j.Logger import org.joml.Vector3f import org.joml.Vector3fc import org.joml.Vector3i -import org.vitrivr.engine.core.model.mesh.Mesh -import org.vitrivr.engine.model3d.util.MeshMathUtil +import org.vitrivr.engine.core.model.mesh.texturemodel.Mesh +import org.vitrivr.engine.core.model.mesh.texturemodel.util.types.Vec3f +import org.vitrivr.engine.model3d.lwjglrender.util.MeshMathUtil import kotlin.math.abs import kotlin.math.ceil import kotlin.math.pow @@ -138,7 +139,7 @@ class Voxelizer(private val resolution: Float) { * @return true if the voxel's center is within the circle, false otherwise. */ private fun vertextTest(vertex: Mesh.Vertex, voxel: Pair): Boolean { - return vertex.position.distanceSquared(voxel.second) > this.rcsq + return vertex.position.distanceSquared(Vec3f(voxel.second.x, voxel.second.y, voxel.second.z)) > this.rcsq } /** @@ -150,8 +151,8 @@ class Voxelizer(private val resolution: Float) { * @return true if voxel's center is contained in cylinder, false otherwise. */ private fun edgeTest(a: Mesh.Vertex, b: Mesh.Vertex, voxel: Pair): Boolean { - val line: Vector3f = Vector3f(b.position).sub(a.position) - val pd = voxel.second.sub(a.position) + val line: Vector3f = Vector3f(b.position.x,b.position.y,b.position.z).sub(Vector3f(a.position.x,a.position.y,a.position.z)) + val pd = voxel.second.sub(Vector3f(a.position.x,a.position.y,a.position.z)) /* Calculate distance between a and b (Edge). */ val lsq: Float = a.position.distanceSquared(b.position) @@ -183,8 +184,8 @@ class Voxelizer(private val resolution: Float) { val vcorner = Vector3f(this.rc, this.rc, this.rc).add(vcenter) /* Calculate the vectors spanning the plane of the facepolyon and its plane-normal. */ - val ab: Vector3f = Vector3f(b.position).sub(a.position) - val ac: Vector3f = Vector3f(c.position).sub(a.position) + val ab: Vector3f = Vector3f(b.position.x,b.position.y,b.position.z).sub(Vector3f(a.position.x,a.position.y,a.position.z)) + val ac: Vector3f = Vector3f(c.position.x,c.position.y,c.position.z).sub(Vector3f(a.position.x,a.position.y,a.position.z)) val planenorm = Vector3f(ab).cross(ac) /* Calculate the distance t for enclosing planes. */ @@ -209,7 +210,7 @@ class Voxelizer(private val resolution: Float) { /* Calculate bounding box for provided vertices. */ val positions = ArrayList(vertices.size) for (vertex in vertices) { - positions.add(vertex.position) + positions.add(Vector3f(vertex.position.x, vertex.position.y, vertex.position.z)) } val bounds: FloatArray = MeshMathUtil.bounds(positions) diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/renderer/ExternalRenderer.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/renderer/ExternalRenderer.kt new file mode 100644 index 000000000..46ff6782f --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/renderer/ExternalRenderer.kt @@ -0,0 +1,151 @@ +package org.vitrivr.engine.model3d.renderer + +import org.joml.Vector3f +import org.vitrivr.engine.core.model.mesh.texturemodel.Model3d +import org.vitrivr.engine.model3d.lwjglrender.render.RenderOptions +import org.vitrivr.engine.model3d.lwjglrender.window.WindowOptions +import java.awt.image.BufferedImage +import java.io.* +import kotlin.jvm.optionals.getOrNull + +/** + * A helper class that boots an external renderer and allows to render 3D models using it. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class ExternalRenderer : Closeable { + + + companion object { + private const val CLASS_NAME = "org.vitrivr.engine.model3d.renderer.RendererKt" + } + + private var process: RenderProcess? = null + + @Volatile + /** Flag indicating whether the [ExternalRenderer] is closed. */ + private var closed: Boolean = false + + /** + * Renders the given [Model3d] using the [ExternalRenderer]. + * + * @param model The [Model3d] to render. + * @param cameraPositions The [List] of [Vector3f] representing the camera positions. + * @param windowOptions The [WindowOptions] to use for rendering. + * @param renderOptions The [RenderOptions] to use for rendering. + */ + @Synchronized + fun render(model: Model3d, cameraPositions: List, windowOptions: WindowOptions, renderOptions: RenderOptions): List { + check(!this.closed) { "ExternalRenderer is closed and cannot be used for processing." } + var process = this.process + if (process == null || !process.isAlive()) { + process?.close() + process = RenderProcess() + this.process = process + } + + /* Send request. */ + val request = RenderRequest(model, cameraPositions, windowOptions, renderOptions) + val response = process.send(request) + + /* Return images . */ + return response.images() + } + + /** + * Closes the [ExternalRenderer] and the associated process. + */ + override fun close() { + if (!this.closed) { + this.closed = true + this.process?.close() + this.process = null + } + } + + /** + * A [RenderProcess] is a helper class that wraps the [Process] used by the [ExternalRenderer]. + */ + private inner class RenderProcess: Closeable { + + /** The [Process] used by the [ExternalRenderer]. */ + val process: Process + + /** The [ObjectOutputStream] used by the [ExternalRenderer]. */ + val oos: ObjectOutputStream + + /** The [ObjectInputStream] used by the [ExternalRenderer]. */ + val ois: ObjectInputStream + + /** The standard error */ + val err: BufferedReader + + init { + val javaBin = ProcessHandle.current().info().command().getOrNull() ?: throw IllegalStateException("Could not determine JAVA_HOME.") + val classpath = System.getProperty("java.class.path") + val os = System.getProperty("os.name").lowercase() + val processBuilder = if (os.contains("mac")) { + ProcessBuilder(javaBin, "-cp", classpath, "-XstartOnFirstThread", CLASS_NAME) /* Mac only issue. */ + } else { + ProcessBuilder(javaBin, "-cp", classpath, CLASS_NAME) + } + if (os.contains("linux")) { + processBuilder.environment()["DISPLAY"] = ":1" + } + this.process = processBuilder.start() + + /* Initialize streams. */ + this.err = this.process.errorReader() + try { + this.oos = ObjectOutputStream(this.process.outputStream) + this.ois = ObjectInputStream(this.process.inputStream) + } catch (e: Throwable) { + val err = this.err.readText() + throw IllegalStateException("Failed to start external renderer due to error: $err", e) + } + } + + /** + * Sends a [RenderRequest] to the external renderer. + */ + fun send(request: RenderRequest): RenderResponse { + /* Write request to output stream. */ + try { + this.oos.writeObject(request) + this.oos.flush() + } catch (e: IOException) { + this.oos.reset() + val err = this.err.readText() + throw IllegalStateException("Could not send request due to IO exception. External renderer reported: $err", e) + } + + /* Read response and return image. */ + val image = try { + this.ois.readObject() as? RenderResponse ?: throw IllegalStateException("Could not parse model.") + } catch (e: IOException) { + this.ois.reset() + val err = this.err.readText() + throw IllegalStateException("Could not parse model due to IO exception. External renderer reported: $err", e) + } + return image + } + + /** + * Checks if the process is still alive. + */ + fun isAlive(): Boolean { + return this.process.isAlive + } + + /** + * Checks if the process is still alive. + */ + override fun close() = try { + this.oos.close() + this.ois.close() + } finally { + this.process.destroy() + } + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/renderer/RenderRequest.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/renderer/RenderRequest.kt new file mode 100644 index 000000000..44caaa6bb --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/renderer/RenderRequest.kt @@ -0,0 +1,61 @@ +package org.vitrivr.engine.model3d.renderer + +import org.joml.Vector3f +import org.vitrivr.engine.core.model.mesh.texturemodel.Model3d +import org.vitrivr.engine.model3d.lwjglrender.render.RenderOptions +import org.vitrivr.engine.model3d.lwjglrender.renderer.RenderActions +import org.vitrivr.engine.model3d.lwjglrender.renderer.RenderData +import org.vitrivr.engine.model3d.lwjglrender.renderer.RenderJob +import org.vitrivr.engine.model3d.lwjglrender.util.datatype.Variant +import org.vitrivr.engine.model3d.lwjglrender.util.fsm.model.Action +import org.vitrivr.engine.model3d.lwjglrender.window.WindowOptions +import java.io.Serializable +import java.util.* +import java.util.concurrent.LinkedBlockingDeque + +/** + * A [RenderRequest] as processed by the [ExternalRenderer]. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +data class RenderRequest(val model: Model3d, val cameraPositions: List, val windowOptions: WindowOptions, val renderOptions: RenderOptions) : Serializable { + + + companion object { + private const val serialVersionUID: Long = 42L + } + + /** + * Converts this [RenderRequest] to a [RenderJob]. + * + * @return [RenderJob] + */ + fun toJob(): RenderJob { + // Create data bag for the job. + val jobData = Variant() + jobData.set(RenderData.WINDOWS_OPTIONS, windowOptions) + .set(RenderData.RENDER_OPTIONS, renderOptions) + .set(RenderData.MODEL, model) + + // Setup the action sequence to perform the jop + // In standard jop, this is an image for each camera position + val actions = LinkedBlockingDeque() + actions.add(Action(RenderActions.SETUP.name)) + actions.add(Action(RenderActions.SETUP.name)) + actions.add(Action(RenderActions.SETUP.name)) + + val vectors = LinkedList() + for (position in cameraPositions) { + // Create a copy of the vector to avoid concurrent modification exceptions + vectors.add(Vector3f(position)) + actions.add(Action(RenderActions.LOOKAT_FROM.name)) + actions.add(Action(RenderActions.RENDER.name)) + } + actions.add(Action(RenderActions.SETUP.name)) + jobData.set(RenderData.VECTORS, vectors) + + // Add the job to the queue + return RenderJob(actions, jobData) + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/renderer/RenderResponse.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/renderer/RenderResponse.kt new file mode 100644 index 000000000..132616464 --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/renderer/RenderResponse.kt @@ -0,0 +1,61 @@ +package org.vitrivr.engine.model3d.renderer + +import java.awt.image.BufferedImage +import java.io.* +import javax.imageio.ImageIO + +/** + * A [RenderResponse] as generated by the [ExternalRenderer]. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +data class RenderResponse(private var list: List = emptyList()) : Serializable { + + companion object { + private const val serialVersionUID: Long = 43L + } + + /** + * Returns all the [BufferedImage]s contained in this [RenderResponse]. + * + * @return [List] of [BufferedImage]s. + */ + fun images(): List { + return this.list + } + + /** + * Writes this [RenderResponse] to an [ObjectOutputStream]. + * + * @param oos [ObjectOutputStream] to write to. + */ + @Throws(IOException::class) + private fun writeObject(oos: ObjectOutputStream) { + oos.writeInt(this.list.size) + for (image in this.list) { + val baos = ByteArrayOutputStream() + ImageIO.write(image, "png", baos) + val bytes = baos.toByteArray() + oos.writeInt(bytes.size) + oos.write(bytes) + } + } + + /** + * Reads this [RenderResponse] from an [ObjectOutputStream]. + * + * @param in [ObjectInputStream] to read from. + */ + @Throws(IOException::class) + private fun readObject(`in`: ObjectInputStream) { + val size = `in`.readInt() + this.list = (0 until size).map { + val length = `in`.readInt() + val bytes = ByteArray(length) + val bais = ByteArrayInputStream(bytes) + `in`.readFully(bytes) + ImageIO.read(bais) + } + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/renderer/Renderer.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/renderer/Renderer.kt new file mode 100644 index 000000000..8e90fc77e --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/renderer/Renderer.kt @@ -0,0 +1,81 @@ +package org.vitrivr.engine.model3d.renderer + +import org.vitrivr.engine.model3d.lwjglrender.renderer.RenderData +import org.vitrivr.engine.model3d.lwjglrender.renderer.RenderWorker +import org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker.JobControlCommand +import org.vitrivr.engine.model3d.lwjglrender.util.fsm.abstractworker.JobType +import java.awt.image.BufferedImage +import java.io.ObjectInputStream +import java.io.ObjectOutputStream +import java.util.* +import java.util.concurrent.LinkedBlockingDeque +import kotlin.system.exitProcess + +/** + * Main method of the external rendering process. + */ +fun main() { + + /* Initialize RenderWorker. */ + val worker = try { + RenderWorker(LinkedBlockingDeque()) + } catch (e: Throwable) { + System.err.println(e.message) + exitProcess(1) + } + + /* Listen for incoming models. */ + ObjectOutputStream(System.out).use { os -> + ObjectInputStream(System.`in`).use { ois -> + while (true) { + val received = ois.readObject() as? RenderRequest + if (received == null) { + System.err.println("Could not parse model.") + os.writeObject(RenderResponse(emptyList())) + continue + } + + /* Perform rendering. */ + val job = received.toJob() + val images = LinkedList() + + /* Perform render job and collect images. */ + try { + worker.performJob(job) + var finished = false + while (!finished) { + val result = job.results + when (result.type) { + JobType.RESPONSE -> { + val result = result.data!!.get(BufferedImage::class.java, RenderData.IMAGE) + if (result is BufferedImage) { + images.add(result) + } + } + + JobType.CONTROL -> { + if (result.command == JobControlCommand.JOB_FAILURE) { + System.err.println("Job failed.") + finished = true + } + if (result.command == JobControlCommand.JOB_DONE) { + finished = true + } + } + + else -> { + /* No op. */ + } + } + } + } catch (e: Throwable) { + System.err.println("Unhandled error during processing: ${e.message}") + } finally { + /* Send back response. */ + os.writeObject(RenderResponse(images)) + os.flush() + } + } + } + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/resources/META-INF/services/org.vitrivr.engine.core.operators.general.ExporterFactory b/vitrivr-engine-module-m3d/src/main/resources/META-INF/services/org.vitrivr.engine.core.operators.general.ExporterFactory new file mode 100644 index 000000000..560c5adc1 --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/resources/META-INF/services/org.vitrivr.engine.core.operators.general.ExporterFactory @@ -0,0 +1 @@ +org.vitrivr.engine.model3d.ModelPreviewExporter \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/resources/renderer/lwjgl/models/default/default.png b/vitrivr-engine-module-m3d/src/main/resources/renderer/lwjgl/models/default/default.png new file mode 100644 index 000000000..909c66db1 Binary files /dev/null and b/vitrivr-engine-module-m3d/src/main/resources/renderer/lwjgl/models/default/default.png differ diff --git a/vitrivr-engine-module-m3d/src/main/resources/renderer/lwjgl/models/unit-cube/Cube_Text.bin b/vitrivr-engine-module-m3d/src/main/resources/renderer/lwjgl/models/unit-cube/Cube_Text.bin new file mode 100644 index 000000000..2f70ed9be Binary files /dev/null and b/vitrivr-engine-module-m3d/src/main/resources/renderer/lwjgl/models/unit-cube/Cube_Text.bin differ diff --git a/vitrivr-engine-module-m3d/src/main/resources/renderer/lwjgl/models/unit-cube/Cube_Text.gltf b/vitrivr-engine-module-m3d/src/main/resources/renderer/lwjgl/models/unit-cube/Cube_Text.gltf new file mode 100644 index 000000000..96715546d --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/resources/renderer/lwjgl/models/unit-cube/Cube_Text.gltf @@ -0,0 +1,143 @@ +{ + "asset" : { + "generator" : "Khronos glTF Blender I/O v3.3.32", + "version" : "2.0" + }, + "scene" : 0, + "scenes" : [ + { + "name" : "Scene", + "nodes" : [ + 0 + ] + } + ], + "nodes" : [ + { + "mesh" : 0, + "name" : "Cube_Cube.001", + "rotation" : [ + 1, + 0, + 0, + 0 + ] + } + ], + "materials" : [ + { + "doubleSided" : true, + "name" : "Material", + "pbrMetallicRoughness" : { + "baseColorTexture" : { + "index" : 0 + }, + "metallicFactor" : 0, + "roughnessFactor" : 0.5 + } + } + ], + "meshes" : [ + { + "name" : "Cube_Cube.006", + "primitives" : [ + { + "attributes" : { + "POSITION" : 0, + "NORMAL" : 1, + "TEXCOORD_0" : 2 + }, + "indices" : 3, + "material" : 0 + } + ] + } + ], + "textures" : [ + { + "sampler" : 0, + "source" : 0 + } + ], + "images" : [ + { + "mimeType" : "image/png", + "name" : "cube", + "uri" : "cube.png" + } + ], + "accessors" : [ + { + "bufferView" : 0, + "componentType" : 5126, + "count" : 33, + "max" : [ + 1, + 1, + 1 + ], + "min" : [ + -1, + -1, + -1 + ], + "type" : "VEC3" + }, + { + "bufferView" : 1, + "componentType" : 5126, + "count" : 33, + "type" : "VEC3" + }, + { + "bufferView" : 2, + "componentType" : 5126, + "count" : 33, + "type" : "VEC2" + }, + { + "bufferView" : 3, + "componentType" : 5123, + "count" : 36, + "type" : "SCALAR" + } + ], + "bufferViews" : [ + { + "buffer" : 0, + "byteLength" : 396, + "byteOffset" : 0, + "target" : 34962 + }, + { + "buffer" : 0, + "byteLength" : 396, + "byteOffset" : 396, + "target" : 34962 + }, + { + "buffer" : 0, + "byteLength" : 264, + "byteOffset" : 792, + "target" : 34962 + }, + { + "buffer" : 0, + "byteLength" : 72, + "byteOffset" : 1056, + "target" : 34963 + } + ], + "samplers" : [ + { + "magFilter" : 9729, + "minFilter" : 9987 + } + ], + "buffers" : [ + { + "byteLength" : 1128, + "uri" : "Cube_Text.bin" + } + ] +} diff --git a/vitrivr-engine-module-m3d/src/main/resources/renderer/lwjgl/models/unit-cube/cube.png b/vitrivr-engine-module-m3d/src/main/resources/renderer/lwjgl/models/unit-cube/cube.png new file mode 100644 index 000000000..0b31bb130 Binary files /dev/null and b/vitrivr-engine-module-m3d/src/main/resources/renderer/lwjgl/models/unit-cube/cube.png differ diff --git a/vitrivr-engine-module-m3d/src/main/resources/renderer/lwjgl/shaders/scene.frag b/vitrivr-engine-module-m3d/src/main/resources/renderer/lwjgl/shaders/scene.frag new file mode 100644 index 000000000..19061a687 --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/resources/renderer/lwjgl/shaders/scene.frag @@ -0,0 +1,18 @@ +#version 330 + +in vec2 outTextCoord; + +out vec4 fragColor; + +struct Material +{ + vec4 diffuse; +}; + +uniform sampler2D txtSampler; +uniform Material material; + +void main() +{ + fragColor = texture(txtSampler, outTextCoord) + material.diffuse; +} \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/resources/renderer/lwjgl/shaders/scene.vert b/vitrivr-engine-module-m3d/src/main/resources/renderer/lwjgl/shaders/scene.vert new file mode 100644 index 000000000..338f34f89 --- /dev/null +++ b/vitrivr-engine-module-m3d/src/main/resources/renderer/lwjgl/shaders/scene.vert @@ -0,0 +1,16 @@ +#version 330 + +layout (location=0) in vec3 position; +layout (location=1) in vec2 texCoord; + +out vec2 outTextCoord; + +uniform mat4 projectionMatrix; +uniform mat4 viewMatrix; +uniform mat4 modelMatrix; + +void main() +{ + gl_Position = projectionMatrix * viewMatrix * modelMatrix * vec4(position, 1.0); + outTextCoord = texCoord; +} \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/test/kotlin/org/vitrivr/engine/model3d/M3DTest.kt b/vitrivr-engine-module-m3d/src/test/kotlin/org/vitrivr/engine/model3d/M3DTest.kt new file mode 100644 index 000000000..861a85937 --- /dev/null +++ b/vitrivr-engine-module-m3d/src/test/kotlin/org/vitrivr/engine/model3d/M3DTest.kt @@ -0,0 +1,50 @@ +package org.vitrivr.engine.model3d + +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test +import org.vitrivr.engine.model3d.renderer.ExternalRenderer + +/** + * Unit tests to check the functionality of the [ModelLoader]. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class M3DTest { + /** + * Tests if the Standford Bunny model can be loaded. + */ + @Test + fun loadBunny() { + /* Load model. */ + val loader = ModelLoader() + val model = this::class.java.getResourceAsStream("/bunny.obj").use { inp -> + loader.loadModel("bunny", inp!!) + } ?: Assertions.fail("Failed to load model.") + + /* Check if model was loaded correctly. */ + Assertions.assertTrue(model.getMaterials().size == 1) + Assertions.assertTrue(model.getAllNormals().size == 4968) + } + + /** + * Tests if the Standford Bunny model can be loaded. + */ + @Test + fun renderBunny() { + /* Load model. */ + val loader = ModelLoader() + val model = this::class.java.getResourceAsStream("/bunny.obj").use { inp -> + loader.loadModel("bunny", inp!!) + } ?: Assertions.fail("Failed to load model.") + + /* Render image. */ + val renderer = ExternalRenderer() + Assertions.assertDoesNotThrow() { + ModelPreviewExporter.renderPreviewJPEG(model, renderer, 1.0f) + } + Assertions.assertDoesNotThrow() { + renderer.close() + } + } +} \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/test/kotlin/org/vitrivr/engine/model3d/ModelHandlerTest.kt b/vitrivr-engine-module-m3d/src/test/kotlin/org/vitrivr/engine/model3d/ModelHandlerTest.kt deleted file mode 100644 index e26e984c5..000000000 --- a/vitrivr-engine-module-m3d/src/test/kotlin/org/vitrivr/engine/model3d/ModelHandlerTest.kt +++ /dev/null @@ -1,29 +0,0 @@ -package org.vitrivr.engine.model3d - -import org.junit.jupiter.api.Assertions -import org.junit.jupiter.api.Test -import java.nio.file.Files -import java.nio.file.StandardOpenOption - -class ModelHandlerTest { - @Test - fun loadBunny() { - val handler = ModelHandler() - val path = Files.createTempFile("bunny", ".obj") - Files.newOutputStream(path, StandardOpenOption.CREATE, StandardOpenOption.WRITE).use { out -> - this::class.java.getResourceAsStream("/bunny.obj").use { inp -> - out.write(inp!!.readAllBytes()) - } - } - - /* Validate model */ - val model = handler.loadModel("test", path.toString()) - Assertions.assertTrue(model.getMaterials().size == 1) - Assertions.assertTrue(model.getMaterials().first().meshes.size == 1) - - /* Validate mesh. */ - val mesh = model.getMaterials().first().meshes.first() - Assertions.assertEquals(4968, mesh.numberOfFaces) - Assertions.assertEquals(2503, mesh.numberOfVertices) - } -} \ No newline at end of file diff --git a/vitrivr-engine-server/src/main/kotlin/org/vitrivr/engine/server/Main.kt b/vitrivr-engine-server/src/main/kotlin/org/vitrivr/engine/server/Main.kt index 865daa868..6af233582 100644 --- a/vitrivr-engine-server/src/main/kotlin/org/vitrivr/engine/server/Main.kt +++ b/vitrivr-engine-server/src/main/kotlin/org/vitrivr/engine/server/Main.kt @@ -89,11 +89,14 @@ fun main(args: Array) { cli.register(SchemaCommand(schema, executor)) } - /* Start the Javalin and CLI. */ + /* Start the Javalin server. */ javalin.start(config.api.port) logger.info { "vitrivr engine API is listening on port ${config.api.port}." } - cli.start() /* Blocks. */ - /* End Javalin once Cli is stopped. */ + /* Start the CLI in a new thread; this will block the thread it runs on. */ + cli.start() + + /* Upon reaching this point, the program was aborted. */ + /* Stop the Javalin server. */ javalin.stop() }